diff --git a/Cargo.lock b/Cargo.lock index d878117..381bfde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1488,6 +1488,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -2583,6 +2603,12 @@ dependencies = [ "litrs", ] +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "dunce" version = "1.0.5" @@ -2956,6 +2982,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8878864ba14bb86e818a412bfd6f18f9eabd4ec0f008a28e8f7eb61db532fcf9" +dependencies = [ + "futures-core", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -4793,6 +4828,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mockall" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58d964098a5f9c6b63d0798e5372fd04708193510a7af313c22e9f29b7b620b" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca41ce716dda6a9be188b385aa78ee5260fc25cd3802cb2a8afdc6afbe6b6dbf" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "modular-bitfield" version = "0.13.1" @@ -4959,6 +5020,8 @@ dependencies = [ "morph-payload-builder", "morph-payload-types", "morph-primitives", + "morph-proofs", + "morph-proofs-exex", "morph-reference-index", "morph-rpc", "morph-txpool", @@ -4983,6 +5046,7 @@ dependencies = [ "reth-prune-types", "reth-rpc-builder", "reth-rpc-eth-api", + "reth-rpc-server-types", "reth-tasks", "reth-tracing", "reth-transaction-pool", @@ -4990,6 +5054,7 @@ dependencies = [ "reth-trie-db", "serde", "serde_json", + "tempfile", "tokio", "tokio-stream", "tracing", @@ -5069,6 +5134,63 @@ dependencies = [ "serde_json", ] +[[package]] +name = "morph-proofs" +version = "1.0.0" +dependencies = [ + "alloy-eips", + "alloy-primitives", + "auto_impl", + "bincode 2.0.1", + "bytes", + "derive_more", + "metrics", + "mockall", + "parking_lot", + "reth-codecs", + "reth-db", + "reth-db-api", + "reth-evm", + "reth-execution-errors", + "reth-metrics", + "reth-primitives-traits", + "reth-provider", + "reth-revm", + "reth-storage-errors", + "reth-tasks", + "reth-trie", + "reth-trie-common", + "serde", + "strum", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "morph-proofs-exex" +version = "1.0.0" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "eyre", + "futures", + "morph-proofs", + "reth-db", + "reth-ethereum-primitives", + "reth-execution-types", + "reth-exex", + "reth-exex-test-utils", + "reth-node-api", + "reth-primitives-traits", + "reth-provider", + "reth-trie", + "tokio", + "tracing", +] + [[package]] name = "morph-reference-index" version = "1.0.0" @@ -5101,17 +5223,26 @@ dependencies = [ name = "morph-reth" version = "1.0.0" dependencies = [ + "alloy-consensus", "alloy-primitives", "clap", + "eyre", "morph-chainspec", "morph-consensus", "morph-evm", "morph-node", + "morph-proofs", + "morph-proofs-exex", + "reth-chainspec", "reth-cli", + "reth-cli-commands", + "reth-cli-runner", "reth-cli-util", "reth-ethereum-cli", "reth-node-builder", + "reth-provider", "reth-rpc-server-types", + "reth-tasks", "tracing", ] @@ -5149,26 +5280,32 @@ dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", "alloy-serde", + "async-trait", "derive_more", "eyre", "jsonrpsee", + "metrics", "morph-chainspec", "morph-evm", "morph-primitives", + "morph-proofs", "morph-reference-index", "morph-revm", "reth-chainspec", "reth-errors", "reth-evm", + "reth-metrics", "reth-node-api", "reth-node-builder", "reth-primitives-traits", "reth-provider", "reth-revm", "reth-rpc", + "reth-rpc-api", "reth-rpc-convert", "reth-rpc-eth-api", "reth-rpc-eth-types", + "reth-rpc-server-types", "reth-storage-api", "reth-tasks", "reth-transaction-pool", @@ -5888,6 +6025,32 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "prefix-trie" version = "0.8.4" @@ -7636,6 +7799,38 @@ dependencies = [ "tracing", ] +[[package]] +name = "reth-exex-test-utils" +version = "2.4.0" +source = "git+https://github.com/paradigmxyz/reth?tag=v2.4.0#943af245c4d69c6c1df241df016c278ffb5d15df" +dependencies = [ + "alloy-eips", + "eyre", + "futures-util", + "reth-chainspec", + "reth-config", + "reth-consensus", + "reth-db", + "reth-db-common", + "reth-ethereum-primitives", + "reth-evm-ethereum", + "reth-execution-types", + "reth-exex", + "reth-network", + "reth-node-api", + "reth-node-builder", + "reth-node-core", + "reth-node-ethereum", + "reth-payload-builder", + "reth-primitives-traits", + "reth-provider", + "reth-tasks", + "reth-transaction-pool", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "reth-exex-types" version = "2.4.0" @@ -7910,7 +8105,7 @@ version = "2.4.0" source = "git+https://github.com/paradigmxyz/reth?tag=v2.4.0#943af245c4d69c6c1df241df016c278ffb5d15df" dependencies = [ "anyhow", - "bincode", + "bincode 1.3.3", "derive_more", "lz4_flex", "memmap2", @@ -10428,6 +10623,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + [[package]] name = "thiserror" version = "1.0.69" @@ -11191,6 +11392,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.8" @@ -11290,6 +11497,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "visibility" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index 6c885d3..7d218b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,8 @@ members = [ "crates/payload/builder", "crates/payload/types", "crates/primitives", + "crates/proofs", + "crates/proofs-exex", "crates/reference-index", "crates/revm", "crates/txpool", @@ -94,6 +96,8 @@ morph-node = { path = "crates/node"} morph-payload-builder = { path = "crates/payload/builder", default-features = false } morph-payload-types = { path = "crates/payload/types", default-features = false } morph-primitives = { path = "crates/primitives", default-features = false } +morph-proofs = { path = "crates/proofs", default-features = false } +morph-proofs-exex = { path = "crates/proofs-exex", default-features = false } morph-reference-index = { path = "crates/reference-index", default-features = false } morph-rpc = { path = "crates/rpc" } morph-revm = { path = "crates/revm", default-features = false } @@ -104,6 +108,7 @@ reth-chain-state = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" reth-chainspec = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-cli = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-cli-commands = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } +reth-cli-runner = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-cli-util = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-codecs = { version = "0.5.0", default-features = false } reth-codecs-derive = "0.5.0" @@ -116,6 +121,9 @@ reth-engine-local = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0 reth-engine-primitives = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-engine-tree = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-errors = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } +reth-exex = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } +reth-exex-test-utils = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } +reth-execution-errors = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-eth-wire-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-ethereum = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-ethereum-cli = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } @@ -146,9 +154,11 @@ reth-rpc-eth-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" reth-rpc-eth-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-rpc-server-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-storage-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } +reth-storage-errors = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0", default-features = false } reth-tasks = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-tracing = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-trie = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } +reth-trie-common = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } reth-zstd-compressors = { version = "0.5.0", default-features = false } reth-execution-cache = { git = "https://github.com/paradigmxyz/reth", tag = "v2.4.0" } @@ -197,6 +207,7 @@ axum = "0.8.4" base64 = "0.22" blake3 = "1.8" bytes = "1.8" +bincode = "2" clap = { version = "4.5.45", features = ["derive"] } const-hex = { version = "1.15.0" } dashmap = "6.0" @@ -210,6 +221,7 @@ itertools = "0.14.0" jiff = { version = "0.2.15", default-features = false } jsonrpsee = { version = "0.26.0", features = ["server", "client", "macros"] } metrics = "0.24.0" +mockall = "0.14" p256 = "0.13" parking_lot = "0.12.4" prometheus-client = "0.24.0" @@ -220,6 +232,7 @@ rand_core = "0.6.4" rayon = "1.10" serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.142" +strum = { version = "0.27", default-features = false } sha2 = "0.10" tempfile = "3.20.0" test-fuzz = "7" diff --git a/README.md b/README.md index a223945..0dd12cf 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Morph Reth is the next-generation execution client for [Morph](https://www.morph - **Morph Hardforks**: Implements Morph hardfork logic through Jade, with bundled Mainnet and Hoodi chainspecs scheduled through Jade - **Custom Engine API**: L2-specific Engine API for sequencer block building and validation - **L1 Fee Validation**: Transaction pool with L1 data fee affordability checks +- **Bounded Historical Proofs**: Optional forward-only EIP-1186 proof history in an independent MDBX database ### Supported Networks @@ -42,6 +43,9 @@ morph-reth/ │ ├── builder/ # Block building logic │ └── types/ # Engine API types (MorphExecutionData, etc.) ├── primitives/ # Core types (transactions, receipts) + ├── proofs/ # Versioned MPT history and proof state provider + ├── proofs-exex/ # Forward accumulation, reorg, unwind, and pruning + ├── reference-index/ # Independent transaction reference index ├── revm/ # L1 fee calculation, token fee logic ├── rpc/ # RPC implementation and type conversions └── txpool/ # Transaction pool with L1 fee validation @@ -60,6 +64,9 @@ morph-reth/ | `morph-payload-types` | Engine API payload types | | `morph-payload-builder` | Block building implementation | | `morph-primitives` | Transaction and receipt types | +| `morph-proofs` | Bounded, versioned MPT history in MDBX | +| `morph-proofs-exex` | Forward-only proof-history execution extension | +| `morph-reference-index` | Independent Morph transaction reference index | | `morph-revm` | L1 fee and token fee calculations | | `morph-rpc` | RPC implementation and type conversions | | `morph-txpool` | Transaction pool with L1 fee and MorphTx validation | @@ -114,7 +121,36 @@ openssl rand -hex 32 > jwt.hex |------|---------|-------------| | `--morph.max-tx-payload-bytes` | 122880 (120KB) | Maximum transaction payload bytes per block | | `--morph.max-tx-per-block` | None (unlimited) | Maximum number of transactions per block | -| `--rpc.eth-proof-window` | 0 (disabled) | Max historical blocks for `eth_getProof` (up to 1209600) | +| `--proofs-history` | false | Enable historical `eth_getProof` and proof-history accumulation | +| `--proofs-history.storage-path` | `/historical-proofs` | Override the proof MDBX directory | +| `--proofs-history.window` | 604800 | Number of canonical blocks retained (7 days at 1s/block) | +| `--proofs-history.verification-interval` | 0 | Re-execute every Nth indexed block; 0 disables verification | +| `--rpc.eth-proof-window` | 0 | Reth overlay limit used only while proof history is disabled | + +#### Initializing Historical Proofs + +Proof history starts at the current canonical tip and accumulates forward. Stop the node before +initialization so the source state remains fixed: + +```bash +./target/release/morph-reth proofs init --chain hoodi --datadir /path/to/reth-data + +./target/release/morph-reth node \ + --chain hoodi \ + --datadir /path/to/reth-data \ + --proofs-history \ + --http \ + --authrpc.jwtsecret jwt.hex +``` + +The standard `eth_getProof` method is then served only for the inclusive range reported by +`debug_proofsSyncStatus`; requests outside that range never fall back to Reth's historical overlay. +The same `eth_getProof` override is installed on the authenticated RPC server. For manual +maintenance use `morph-reth proofs prune` and `morph-reth proofs unwind --target `. + +For cold copies, stop the source node and copy the complete chain data directory, including +`historical-proofs`, as one consistent unit. Startup validates the proof database schema, chain ID, +genesis hash, and latest canonical block hash before resuming. ### Running Tests diff --git a/bin/morph-reth/Cargo.toml b/bin/morph-reth/Cargo.toml index 485617c..63b0430 100644 --- a/bin/morph-reth/Cargo.toml +++ b/bin/morph-reth/Cargo.toml @@ -20,17 +20,26 @@ morph-chainspec = { workspace = true, features = ["cli"] } morph-consensus.workspace = true morph-evm.workspace = true morph-node.workspace = true +morph-proofs.workspace = true +morph-proofs-exex.workspace = true # Reth CLI +reth-chainspec.workspace = true reth-cli.workspace = true +reth-cli-commands.workspace = true +reth-cli-runner.workspace = true reth-cli-util.workspace = true reth-ethereum-cli.workspace = true reth-node-builder.workspace = true +reth-provider.workspace = true reth-rpc-server-types.workspace = true +reth-tasks.workspace = true # Other +alloy-consensus.workspace = true alloy-primitives.workspace = true clap.workspace = true +eyre.workspace = true tracing.workspace = true [features] diff --git a/bin/morph-reth/src/main.rs b/bin/morph-reth/src/main.rs index a39d8ad..207597f 100644 --- a/bin/morph-reth/src/main.rs +++ b/bin/morph-reth/src/main.rs @@ -3,6 +3,8 @@ //! This is the main entry point for the Morph L2 execution layer client. //! It extends reth with Morph-specific functionality. +mod proofs; + #[global_allocator] static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator(); @@ -21,6 +23,10 @@ use morph_chainspec::{MORPH_DEFAULT_PRIORITY_FEE, MorphChainSpec, MorphChainSpec use morph_consensus::MorphConsensus; use morph_evm::{MorphEvmConfig, evm::MorphEvmFactory}; use morph_node::{MorphAddOns, MorphArgs, MorphNode}; +use morph_proofs::{MdbxProofsStorage, MorphProofsStorage, MorphProofsStore, ProofDbIdentity}; +use morph_proofs_exex::MorphProofsExEx; +use proofs::MorphSubcommand; +use reth_chainspec::EthChainSpec; use reth_cli_util::sigsegv_handler; use reth_ethereum_cli::{Cli, Commands}; use reth_node_builder::Node; @@ -33,7 +39,7 @@ fn morph_default_suggested_fee() -> U256 { } fn apply_morph_cli_defaults( - cli: &mut Cli, + cli: &mut Cli, ) { if let Commands::Node(command) = &mut cli.command { command @@ -67,7 +73,8 @@ fn main() { ) }; - let mut cli = Cli::::parse(); + let mut cli = + Cli::::parse(); apply_morph_cli_defaults(&mut cli); // Run the node @@ -75,14 +82,76 @@ fn main() { cli.run_with_components::(components, async move |builder, morph_args| { info!(target: "morph::cli", "Starting Morph-Reth node"); + let chain_spec = builder.config().chain.clone(); + let datadir = builder.config().datadir(); + let chain_id = chain_spec.chain().id(); + let genesis_hash = chain_spec.genesis_hash(); + + let proof_history = if morph_args.proofs_history { + let path = morph_args + .proofs_history_storage_path + .clone() + .unwrap_or_else(|| datadir.data_dir().join("historical-proofs")); + info!( + target: "morph::proofs", + path = %path.display(), + chain_id, + "opening Morph historical proof database" + ); + let storage: MorphProofsStorage> = Arc::new( + MdbxProofsStorage::open( + &path, + ProofDbIdentity::new(chain_id, genesis_hash), + ) + .map_err(|error| { + eyre::eyre!( + "failed to open historical proof database at {}: {error}", + path.display() + ) + })?, + ); + if storage.get_earliest_block_number()?.is_none() + || storage.get_latest_block_number()?.is_none() + { + return Err(eyre::eyre!( + "proof history is enabled but {} is not initialized; run `morph-reth proofs init` first", + path.display() + )); + } + Some(( + storage, + morph_args.proofs_history_window, + morph_args.proofs_history_verification_interval, + )) + } else { + None + }; let node = MorphNode::new(morph_args); - let handle = builder + let mut add_ons = MorphAddOns::new(); + if let Some((storage, _, _)) = &proof_history { + add_ons = add_ons.with_proof_history(storage.clone()); + } + + let mut node_builder = builder .with_types::() .with_components(node.components_builder()) - .with_add_ons(MorphAddOns::new()) - .launch_with_debug_capabilities() - .await?; + .with_add_ons(add_ons); + + if let Some((storage, window, verification_interval)) = proof_history { + node_builder = node_builder.install_exex( + "morph-proof-history", + async move |ctx| { + let exex = MorphProofsExEx::builder(ctx, storage) + .with_proofs_history_window(window) + .with_verification_interval(verification_interval) + .build(); + Ok(async move { exex.run().await }) + }, + ); + } + + let handle = node_builder.launch_with_debug_capabilities().await?; info!(target: "morph::cli", "Node started successfully"); @@ -94,3 +163,32 @@ fn main() { std::process::exit(1); } } + +#[cfg(test)] +mod tests { + use super::*; + + type MorphCli = + Cli; + + #[test] + fn parses_top_level_proofs_command() { + let cli = MorphCli::try_parse_from(["morph-reth", "proofs", "init", "--chain", "hoodi"]) + .expect("top-level proofs command must parse"); + assert!(matches!( + cli.command, + Commands::Ext(MorphSubcommand::Proofs(_)) + )); + } + + #[test] + fn disabled_mode_keeps_reth_historical_overlay_off() { + let cli = MorphCli::try_parse_from(["morph-reth", "node", "--chain", "hoodi"]) + .expect("node command must parse"); + let Commands::Node(command) = cli.command else { + panic!("expected node command") + }; + assert!(!command.ext.proofs_history); + assert_eq!(command.rpc.rpc_eth_proof_window, 0); + } +} diff --git a/bin/morph-reth/src/proofs.rs b/bin/morph-reth/src/proofs.rs new file mode 100644 index 0000000..5b9b999 --- /dev/null +++ b/bin/morph-reth/src/proofs.rs @@ -0,0 +1,254 @@ +//! Management commands for the independent historical-proof database. + +use std::{path::PathBuf, sync::Arc}; + +use alloy_consensus::BlockHeader; +use clap::{Parser, Subcommand}; +use morph_chainspec::{MorphChainSpec, MorphChainSpecParser}; +use morph_node::MorphNode; +use morph_proofs::{ + DEFAULT_PROOFS_HISTORY_WINDOW, InitializationJob, MdbxProofsStorage, MorphProofStoragePruner, + MorphProofsStorage, MorphProofsStore, ProofDbIdentity, +}; +use reth_chainspec::EthChainSpec; +use reth_cli_commands::common::{AccessRights, Environment, EnvironmentArgs}; +use reth_cli_runner::CliRunner; +use reth_ethereum_cli::ExtendedCommand; +use reth_provider::{ + BlockNumReader, BlockReader, DBProvider, DatabaseProviderFactory, TransactionVariant, +}; +use tracing::{info, warn}; + +/// Morph-specific top-level CLI commands. +#[derive(Debug, Subcommand)] +pub(crate) enum MorphSubcommand { + /// Manage the historical proof database. + #[command(name = "proofs")] + Proofs(ProofsCommand), +} + +impl ExtendedCommand for MorphSubcommand { + fn execute(self, runner: CliRunner) -> eyre::Result<()> { + match self { + Self::Proofs(command) => { + let runtime = runner.runtime(); + runner.run_blocking_until_ctrl_c(command.execute(runtime)) + } + } + } +} + +/// Manage Morph historical proofs. +#[derive(Debug, Parser)] +pub(crate) struct ProofsCommand { + #[command(subcommand)] + command: ProofsSubcommand, +} + +impl ProofsCommand { + async fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + match self.command { + ProofsSubcommand::Init(command) => command.execute(runtime), + ProofsSubcommand::Prune(command) => command.execute(runtime), + ProofsSubcommand::Unwind(command) => command.execute(runtime), + } + } +} + +#[derive(Debug, Subcommand)] +enum ProofsSubcommand { + /// Initialize proof history from the current canonical tip. + Init(InitCommand), + /// Remove proof history outside the configured window. + Prune(PruneCommand), + /// Remove proof history at and after a canonical block. + Unwind(UnwindCommand), +} + +#[derive(Debug, Parser)] +struct InitCommand { + #[command(flatten)] + env: EnvironmentArgs, + + /// Proof-history MDBX directory (defaults to `/historical-proofs`). + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + storage_path: Option, +} + +impl InitCommand { + fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + let Environment { + provider_factory, + data_dir, + .. + } = self.env.init::(AccessRights::RO, runtime)?; + let path = self + .storage_path + .unwrap_or_else(|| data_dir.data_dir().join("historical-proofs")); + let storage = open_storage(&path, &self.env.chain)?; + + if let Some((number, hash)) = storage.get_earliest_block_number()? { + info!(target: "morph::proofs", block_number = number, ?hash, "Proof history is already initialized"); + return Ok(()); + } + + let chain_info = provider_factory.chain_info()?; + info!( + target: "morph::proofs", + path = %path.display(), + block_number = chain_info.best_number, + block_hash = ?chain_info.best_hash, + "Initializing proof history from canonical state" + ); + + let provider = provider_factory + .database_provider_ro()? + .disable_long_read_transaction_safety(); + InitializationJob::new(storage, provider.into_tx()) + .run(chain_info.best_number, chain_info.best_hash)?; + + info!(target: "morph::proofs", "Proof history initialized"); + Ok(()) + } +} + +#[derive(Debug, Parser)] +struct PruneCommand { + #[command(flatten)] + env: EnvironmentArgs, + + /// Proof-history MDBX directory (defaults to `/historical-proofs`). + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + storage_path: Option, + + /// Number of canonical blocks to retain. + #[arg( + long = "proofs-history.window", + value_name = "BLOCKS", + default_value_t = DEFAULT_PROOFS_HISTORY_WINDOW, + value_parser = clap::value_parser!(u64).range(1..) + )] + window: u64, + + /// Maximum number of blocks pruned per transaction. + #[arg( + long = "proofs-history.prune-batch-size", + value_name = "BLOCKS", + default_value_t = 1_000, + value_parser = clap::value_parser!(u64).range(1..) + )] + batch_size: u64, +} + +impl PruneCommand { + fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + let Environment { + provider_factory, + data_dir, + .. + } = self.env.init::(AccessRights::RO, runtime)?; + let path = self + .storage_path + .unwrap_or_else(|| data_dir.data_dir().join("historical-proofs")); + let storage = open_storage(&path, &self.env.chain)?; + + let output = + MorphProofStoragePruner::new(storage, provider_factory, self.window, self.batch_size) + .try_run()?; + info!(target: "morph::proofs", ?output, "Proof-history pruning completed"); + Ok(()) + } +} + +#[derive(Debug, Parser)] +struct UnwindCommand { + #[command(flatten)] + env: EnvironmentArgs, + + /// Proof-history MDBX directory (defaults to `/historical-proofs`). + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + storage_path: Option, + + /// First block to remove, inclusive. + #[arg(long, value_name = "BLOCK")] + target: u64, +} + +impl UnwindCommand { + fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + let Environment { + provider_factory, + data_dir, + .. + } = self.env.init::(AccessRights::RO, runtime)?; + let path = self + .storage_path + .unwrap_or_else(|| data_dir.data_dir().join("historical-proofs")); + let storage = open_storage(&path, &self.env.chain)?; + + let (Some((earliest, _)), Some((latest, _))) = ( + storage.get_earliest_block_number()?, + storage.get_latest_block_number()?, + ) else { + warn!(target: "morph::proofs", "Proof history is empty; nothing to unwind"); + return Ok(()); + }; + + if self.target <= earliest || self.target > latest { + return Err(eyre::eyre!( + "unwind target {} must be in ({earliest}, {latest}]", + self.target + )); + } + + let block = provider_factory + .recovered_block(self.target.into(), TransactionVariant::NoHash)? + .ok_or_else(|| eyre::eyre!("canonical block {} not found", self.target))?; + info!( + target: "morph::proofs", + block_number = block.number(), + block_hash = ?block.hash(), + "Unwinding proof history" + ); + storage.unwind_history(block.block_with_parent())?; + Ok(()) + } +} + +fn open_storage( + path: &std::path::Path, + chain: &MorphChainSpec, +) -> eyre::Result>> { + let identity = ProofDbIdentity::new(chain.chain().id(), chain.genesis_hash()); + let backend = MdbxProofsStorage::open(path, identity).map_err(|error| { + eyre::eyre!( + "failed to open proof history at {}: {error}", + path.display() + ) + })?; + Ok(Arc::new(backend)) +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::ProofsCommand; + + #[test] + fn parses_all_proof_commands() { + ProofsCommand::try_parse_from(["proofs", "init", "--chain", "hoodi"]) + .expect("init command must parse"); + ProofsCommand::try_parse_from([ + "proofs", + "prune", + "--chain", + "hoodi", + "--proofs-history.window", + "64", + ]) + .expect("prune command must parse"); + ProofsCommand::try_parse_from(["proofs", "unwind", "--chain", "hoodi", "--target", "42"]) + .expect("unwind command must parse"); + } +} diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 6c26a46..620180d 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -18,6 +18,7 @@ morph-evm.workspace = true morph-payload-builder.workspace = true morph-payload-types.workspace = true morph-primitives = { workspace = true, features = ["reth-codec"] } +morph-proofs.workspace = true morph-reference-index.workspace = true morph-rpc.workspace = true morph-txpool.workspace = true @@ -85,6 +86,7 @@ reth-db = { workspace = true, features = ["test-utils"] } reth-e2e-test-utils.workspace = true reth-node-core.workspace = true reth-payload-primitives.workspace = true +reth-rpc-server-types.workspace = true reth-tasks.workspace = true reth-tracing.workspace = true alloy-consensus.workspace = true @@ -95,7 +97,9 @@ jsonrpsee.workspace = true morph-chainspec.workspace = true morph-payload-types.workspace = true morph-primitives.workspace = true +morph-proofs-exex.workspace = true serde_json.workspace = true +tempfile.workspace = true [[test]] name = "it" diff --git a/crates/node/src/add_ons.rs b/crates/node/src/add_ons.rs index efe40b0..7dba06d 100644 --- a/crates/node/src/add_ons.rs +++ b/crates/node/src/add_ons.rs @@ -8,9 +8,12 @@ use alloy_hardforks::ForkCondition; use morph_chainspec::{MorphHardfork, MorphHardforks}; use morph_evm::MorphEvmConfig; use morph_primitives::{Block, MorphHeader, MorphReceipt}; +use morph_proofs::{MdbxProofsStorage, MorphProofsStorage}; use morph_reference_index::{ReferenceIndexConfig, ReferenceIndexRuntime}; use morph_rpc::{ - MorphEthApiBuilder, MorphEthConfigApiServer, MorphEthConfigHandler, + MorphEthApiBuilder, MorphEthConfigApiServer, MorphEthConfigHandler, ProofStatusApiExt, + ProofStatusApiOverrideServer, + eth::proofs::{EthProofApiExt, EthProofApiOverrideServer}, morph::{MorphRpc, MorphRpcHandler, MorphRpcServer}, }; use reth_chain_state::CanonStateSubscriptions; @@ -30,6 +33,7 @@ use reth_prune_types::PruneMode; use reth_rpc_builder::Identity; use reth_rpc_eth_api::RpcNodeCore; use reth_tracing::tracing; +use std::sync::Arc; /// Morph node add-ons for RPC and Engine API. /// @@ -48,6 +52,8 @@ pub struct MorphAddOns< > { /// Inner RPC add-ons from reth. inner: RpcAddOns, + /// Optional proof-history storage used to replace `eth_getProof` on normal and auth RPC. + proof_history: Option>>, } impl MorphAddOns, MorphEthApiBuilder> @@ -69,8 +75,18 @@ where Identity::default(), Identity::default(), ), + proof_history: None, } } + + /// Attach initialized proof-history storage to the RPC add-ons. + pub fn with_proof_history( + mut self, + storage: MorphProofsStorage>, + ) -> Self { + self.proof_history = Some(storage); + self + } } fn ensure_reference_index_pruning_compatible( @@ -173,6 +189,7 @@ where let morph_rpc_ctx = MorphRpc::new(reference_index_handle, provider.clone()); let reference_rpc_handler = MorphRpcHandler::new(morph_rpc_ctx); + let proof_history = self.proof_history; // Use launch_add_ons_with to register custom Engine API and eth_config self.inner @@ -180,9 +197,34 @@ where let reth_node_builder::rpc::RpcModuleContainer { modules, auth_module, + registry, .. } = container; + if let Some(storage) = proof_history { + let eth_api = registry.eth_api().clone(); + modules + .replace_configured( + EthProofApiExt::new(eth_api.clone(), storage.clone()).into_rpc(), + ) + .map_err(|error| { + eyre::eyre!("Failed to replace normal RPC eth_getProof: {error}") + })?; + auth_module + .replace_auth_methods( + EthProofApiExt::new(eth_api, storage.clone()).into_rpc(), + ) + .map_err(|error| { + eyre::eyre!("Failed to replace auth RPC eth_getProof: {error}") + })?; + modules + .replace_configured(ProofStatusApiExt::new(storage).into_rpc()) + .map_err(|error| { + eyre::eyre!("Failed to register debug_proofsSyncStatus: {error}") + })?; + tracing::info!(target: "morph::node", "Historical proof RPCs registered"); + } + // Register Morph eth_config handler (EIP-7910 + morph extension) // This provides eth_config on HTTP/WS/IPC for morphnode compatibility. tracing::debug!(target: "morph::node", "Registering Morph eth_config handler"); diff --git a/crates/node/src/args.rs b/crates/node/src/args.rs index 7e5e923..b68657c 100644 --- a/crates/node/src/args.rs +++ b/crates/node/src/args.rs @@ -1,6 +1,9 @@ //! Morph node CLI arguments. +use std::path::PathBuf; + use clap::Args; +use morph_proofs::DEFAULT_PROOFS_HISTORY_WINDOW; /// Default maximum transaction payload bytes per block (120KB). /// @@ -33,6 +36,33 @@ pub struct MorphArgs { /// Morph Holesky testnet uses 1000 as the default limit. #[arg(long = "morph.max-tx-per-block", value_name = "COUNT")] pub max_tx_per_block: Option, + + /// Enable the forward-only historical proof index and its RPC overrides. + #[arg(long = "proofs-history", default_value_t = false)] + pub proofs_history: bool, + + /// Proof-history MDBX directory. + /// + /// Defaults to `/historical-proofs` when proof history is enabled. + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + pub proofs_history_storage_path: Option, + + /// Number of canonical blocks retained in proof history. + #[arg( + long = "proofs-history.window", + value_name = "BLOCKS", + default_value_t = DEFAULT_PROOFS_HISTORY_WINDOW, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub proofs_history_window: u64, + + /// Re-execute every Nth block to verify notification trie data; zero disables verification. + #[arg( + long = "proofs-history.verification-interval", + value_name = "BLOCKS", + default_value_t = 0 + )] + pub proofs_history_verification_interval: u64, } impl Default for MorphArgs { @@ -40,6 +70,10 @@ impl Default for MorphArgs { Self { max_tx_payload_bytes: MORPH_DEFAULT_MAX_TX_PAYLOAD_BYTES, max_tx_per_block: None, + proofs_history: false, + proofs_history_storage_path: None, + proofs_history_window: DEFAULT_PROOFS_HISTORY_WINDOW, + proofs_history_verification_interval: 0, } } } @@ -63,6 +97,10 @@ mod tests { MORPH_DEFAULT_MAX_TX_PAYLOAD_BYTES ); assert_eq!(args.max_tx_per_block, None); + assert!(!args.proofs_history); + assert_eq!(args.proofs_history_storage_path, None); + assert_eq!(args.proofs_history_window, DEFAULT_PROOFS_HISTORY_WINDOW); + assert_eq!(args.proofs_history_verification_interval, 0); } #[test] @@ -105,6 +143,29 @@ mod tests { ); } + #[test] + fn test_proofs_history_args() { + let args = CommandParser::::parse_from([ + "test", + "--proofs-history", + "--proofs-history.storage-path", + "/tmp/morph-proofs", + "--proofs-history.window", + "2592000", + "--proofs-history.verification-interval", + "64", + ]) + .args; + + assert!(args.proofs_history); + assert_eq!( + args.proofs_history_storage_path.as_deref(), + Some(std::path::Path::new("/tmp/morph-proofs")) + ); + assert_eq!(args.proofs_history_window, 2_592_000); + assert_eq!(args.proofs_history_verification_interval, 64); + } + #[test] fn test_default_trait_impl() { let args = MorphArgs::default(); diff --git a/crates/node/src/node.rs b/crates/node/src/node.rs index 347cef9..1f6d788 100644 --- a/crates/node/src/node.rs +++ b/crates/node/src/node.rs @@ -252,6 +252,7 @@ mod tests { let args = super::super::args::MorphArgs { max_tx_payload_bytes: 200_000, max_tx_per_block: Some(500), + ..Default::default() }; let node = MorphNode::new(args); assert_eq!(node.args.max_tx_payload_bytes, 200_000); diff --git a/crates/node/tests/it/main.rs b/crates/node/tests/it/main.rs index fd66dea..66388bc 100644 --- a/crates/node/tests/it/main.rs +++ b/crates/node/tests/it/main.rs @@ -13,6 +13,7 @@ mod evm; mod hardfork; mod l1_messages; mod morph_tx; +mod proof_history; mod reference_index; mod rpc; mod sync; diff --git a/crates/node/tests/it/proof_history.rs b/crates/node/tests/it/proof_history.rs new file mode 100644 index 0000000..2e8a95f --- /dev/null +++ b/crates/node/tests/it/proof_history.rs @@ -0,0 +1,178 @@ +//! End-to-end coexistence coverage for proof history and the reference index. + +use std::{sync::Arc, time::Duration}; + +use alloy_genesis::Genesis; +use alloy_primitives::{Address, B256}; +use alloy_rpc_types_eth::EIP1186AccountProofResponse; +use jsonrpsee::{core::client::ClientT, http_client::HttpClient, rpc_params}; +use morph_chainspec::MorphChainSpec; +use morph_node::{ + MorphAddOns, MorphNode, + test_utils::{MorphTestNode, advance_empty_block, morph_payload_attributes}, +}; +use morph_proofs::{InitializationJob, MdbxProofsStorage, MorphProofsStorage, ProofDbIdentity}; +use morph_proofs_exex::MorphProofsExEx; +use morph_reference_index::ReferenceTransactionResult; +use morph_rpc::{ProofsSyncStatus, ReferenceQueryArgs}; +use reth_chainspec::EthChainSpec; +use reth_e2e_test_utils::node::NodeTestContext; +use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig, NodeHandle}; +use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; +use reth_provider::{DBProvider, DatabaseProviderFactory, providers::BlockchainProvider}; +use reth_rpc_server_types::RpcModuleSelection; +use reth_tasks::Runtime; + +async fn launch_node_with_proof_history( + chain_spec: Arc, + storage: MorphProofsStorage>, +) -> eyre::Result { + let runtime = Runtime::test(); + let network = NetworkArgs { + discovery: DiscoveryArgs { + disable_discovery: true, + ..DiscoveryArgs::default() + }, + ..NetworkArgs::default() + }; + let node_config = NodeConfig::new(chain_spec.clone()) + .with_network(network) + .with_unused_ports() + .with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All), + ); + let node = MorphNode::default(); + let exex_storage = storage.clone(); + + let NodeHandle { + node, + node_exit_future: _, + } = NodeBuilder::new(node_config) + .testing_node(runtime.clone()) + .with_types_and_provider::>() + .with_components(node.components_builder()) + .with_add_ons(MorphAddOns::new().with_proof_history(storage)) + .install_exex("morph-proof-history", async move |ctx| { + let head = ctx.head; + let provider = ctx + .provider() + .database_provider_ro()? + .disable_long_read_transaction_safety(); + InitializationJob::new(exex_storage.clone(), provider.into_tx()) + .run(head.number, head.hash)?; + + let exex = MorphProofsExEx::builder(ctx, exex_storage).build(); + Ok(async move { exex.run().await }) + }) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + reth_node_api::TreeConfig::default().with_cross_block_cache_size(1024 * 1024), + ); + builder.launch_with(launcher) + }) + .await?; + + let node = NodeTestContext::new(node, morph_payload_attributes).await?; + let genesis_hash = chain_spec.genesis_hash(); + node.update_forkchoice(genesis_hash, genesis_hash).await?; + Ok(node) +} + +async fn wait_for_proof_tip(client: &HttpClient, expected: u64) -> eyre::Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut last_status = None; + while tokio::time::Instant::now() < deadline { + if let Ok(status) = client + .request::("debug_proofsSyncStatus", rpc_params![]) + .await + { + if status.latest == Some(expected) { + return Ok(status); + } + last_status = Some(status); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(eyre::eyre!( + "proof history did not reach block {expected}: {last_status:?}" + )) +} + +async fn wait_for_reference_index(client: &HttpClient) -> eyre::Result<()> { + let args = ReferenceQueryArgs { + reference: B256::repeat_byte(0x99), + offset: None, + limit: None, + }; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut last_error = None; + while tokio::time::Instant::now() < deadline { + match client + .request::, _>( + "morph_getTransactionHashesByReference", + (args.clone(),), + ) + .await + { + Ok(results) if results.is_empty() => return Ok(()), + Ok(results) => { + return Err(eyre::eyre!( + "unexpected reference-index results: {results:?}" + )); + } + Err(error) => last_error = Some(error.to_string()), + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(eyre::eyre!( + "reference index did not become ready: {}", + last_error.as_deref().unwrap_or("no RPC response") + )) +} + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_and_reference_index_advance_together() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let genesis: Genesis = serde_json::from_str(include_str!("../assets/test-genesis.json"))?; + let chain_spec = Arc::new(MorphChainSpec::from_genesis(genesis)); + let proof_directory = tempfile::tempdir()?; + let proof_storage = Arc::new(MdbxProofsStorage::open( + proof_directory.path(), + ProofDbIdentity::new(chain_spec.chain().id(), chain_spec.genesis_hash()), + )?); + let mut node = launch_node_with_proof_history(chain_spec, proof_storage).await?; + + advance_empty_block(&mut node).await?; + let client = node + .rpc_client() + .ok_or_else(|| eyre::eyre!("HTTP RPC client not available"))?; + + let status = wait_for_proof_tip(&client, 1).await?; + assert_eq!(status.earliest, Some(0)); + wait_for_reference_index(&client).await?; + + let normal_proof: EIP1186AccountProofResponse = client + .request( + "eth_getProof", + rpc_params![Address::ZERO, Vec::::new(), "0x0"], + ) + .await?; + let auth_proof: EIP1186AccountProofResponse = node + .auth_server_handle() + .http_client() + .request( + "eth_getProof", + rpc_params![Address::ZERO, Vec::::new(), "0x0"], + ) + .await?; + assert_eq!(normal_proof.address, Address::ZERO); + assert_eq!(normal_proof, auth_proof); + + Ok(()) +} diff --git a/crates/proofs-exex/Cargo.toml b/crates/proofs-exex/Cargo.toml new file mode 100644 index 0000000..6c1a672 --- /dev/null +++ b/crates/proofs-exex/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "morph-proofs-exex" +description = "Forward-only proof-history execution extension for Morph" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +morph-proofs.workspace = true + +alloy-consensus.workspace = true +alloy-eips.workspace = true +eyre.workspace = true +futures.workspace = true +reth-execution-types.workspace = true +reth-exex.workspace = true +reth-node-api.workspace = true +reth-provider.workspace = true +reth-trie.workspace = true +tokio.workspace = true +tracing.workspace = true + +[dev-dependencies] +alloy-primitives.workspace = true +morph-proofs = { workspace = true, features = ["test-utils"] } +reth-db = { workspace = true, features = ["test-utils"] } +reth-ethereum-primitives = { workspace = true, features = ["test-utils"] } +reth-exex-test-utils.workspace = true +reth-primitives-traits = { workspace = true, features = ["test-utils"] } + +[features] +default = [] diff --git a/crates/proofs-exex/NOTICE.md b/crates/proofs-exex/NOTICE.md new file mode 100644 index 0000000..db03f0a --- /dev/null +++ b/crates/proofs-exex/NOTICE.md @@ -0,0 +1,31 @@ +# Attribution + +This execution extension is adapted from Base at commit +`b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` under the MIT license: + + + +Morph-specific changes include MDBX-only storage, canonical-tip validation, Morph CLI +messages, and Morph metrics/logging names. + +## Base MIT License + +Copyright (c) 2025 Base + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/proofs-exex/src/lib.rs b/crates/proofs-exex/src/lib.rs new file mode 100644 index 0000000..03b5131 --- /dev/null +++ b/crates/proofs-exex/src/lib.rs @@ -0,0 +1,1190 @@ +//! Forward-only proof-history execution extension for Morph. +//! +//! Adapted from Base commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` +//! under the MIT license. + +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +mod sync_target; +use std::{sync::Arc, time::Duration}; + +use alloy_consensus::BlockHeader; +use alloy_eips::eip1898::BlockWithParent; +use futures::TryStreamExt; +use morph_proofs::{ + DEFAULT_PROOFS_HISTORY_WINDOW, MorphProofStoragePrunerTask, MorphProofsBatchStore, + MorphProofsStorage, + live::{BatchBlock, LiveTrieCollector}, + metrics::ProofHistoryMetrics, +}; +use reth_execution_types::Chain; +use reth_exex::{ExExContext, ExExEvent, ExExNotification, ExExNotificationsStream}; +use reth_node_api::{FullNodeComponents, NodePrimitives, NodeTypes}; +use reth_provider::{BlockHashReader, BlockNumReader, BlockReader, TransactionVariant}; +pub use sync_target::{CachedBlockTrieData, SyncTarget, SyncTargetState}; +use tokio::task; +use tracing::{debug, error, info}; + +/// Default safety threshold for the gap between stored earliest block and the configured +/// window target. When exceeded on startup, the node refuses to auto-prune and asks the +/// operator to run `proofs prune` manually. The threshold is a backstop against accidental +/// misconfiguration (e.g. shrinking the proofs window by orders of magnitude) — it is not a +/// measure of pruner throughput. Override via +/// [`MorphProofsExExBuilder::with_max_prune_blocks_startup`]. +const DEFAULT_MAX_PRUNE_BLOCKS_STARTUP: u64 = 1_000; + +/// How many blocks to process in a single sync turn before yielding. +const SYNC_BLOCKS_PER_TURN: usize = 50; + +/// Default interval between proof-storage prune runs. Default is 15 seconds. +const DEFAULT_PRUNE_INTERVAL: Duration = Duration::from_secs(15); + +/// Default verification interval: disabled +const DEFAULT_VERIFICATION_INTERVAL: u64 = 0; // disabled + +/// Builder for [`MorphProofsExEx`]. +#[derive(Debug)] +pub struct MorphProofsExExBuilder +where + Node: FullNodeComponents, +{ + ctx: ExExContext, + storage: MorphProofsStorage, + proofs_history_window: u64, + proofs_history_prune_interval: Duration, + verification_interval: u64, + max_prune_blocks_startup: u64, +} + +impl MorphProofsExExBuilder +where + Node: FullNodeComponents, +{ + /// Create a new builder with required parameters and defaults. + pub const fn new(ctx: ExExContext, storage: MorphProofsStorage) -> Self { + Self { + ctx, + storage, + proofs_history_window: DEFAULT_PROOFS_HISTORY_WINDOW, + proofs_history_prune_interval: DEFAULT_PRUNE_INTERVAL, + verification_interval: DEFAULT_VERIFICATION_INTERVAL, + max_prune_blocks_startup: DEFAULT_MAX_PRUNE_BLOCKS_STARTUP, + } + } + + /// Sets the window to span blocks for proofs history. + pub const fn with_proofs_history_window(mut self, window: u64) -> Self { + self.proofs_history_window = window; + self + } + + /// Sets the interval between proof-storage prune runs. + pub const fn with_proofs_history_prune_interval(mut self, interval: Duration) -> Self { + self.proofs_history_prune_interval = interval; + self + } + + /// Sets the verification interval. + pub const fn with_verification_interval(mut self, interval: u64) -> Self { + self.verification_interval = interval; + self + } + + /// Sets the safety threshold for blocks that may be auto-pruned at startup. + /// See `DEFAULT_MAX_PRUNE_BLOCKS_STARTUP`. + pub const fn with_max_prune_blocks_startup(mut self, max_blocks: u64) -> Self { + self.max_prune_blocks_startup = max_blocks; + self + } + + /// Builds the [`MorphProofsExEx`]. + pub fn build(self) -> MorphProofsExEx { + MorphProofsExEx { + ctx: self.ctx, + storage: self.storage, + proofs_history_window: self.proofs_history_window, + proofs_history_prune_interval: self.proofs_history_prune_interval, + verification_interval: self.verification_interval, + max_prune_blocks_startup: self.max_prune_blocks_startup, + } + } +} + +/// Proofs `ExEx` - processes blocks and tracks state changes within fault proof window. +/// +/// Saves and serves trie nodes to make proofs faster. This handles the process of +/// saving the current state, new blocks as they're added, and serving proof RPCs +/// based on the saved data. +/// +#[derive(Debug)] +pub struct MorphProofsExEx +where + Node: FullNodeComponents, +{ + /// The `ExEx` context containing the node related utilities e.g. provider, notifications, + /// events. + ctx: ExExContext, + /// The type of storage DB. + storage: MorphProofsStorage, + /// The window to span blocks for proofs history. Value is the number of blocks, received as + /// cli arg. + proofs_history_window: u64, + /// Interval between proof-storage prune runs + proofs_history_prune_interval: Duration, + /// Verification interval: perform full block execution every N blocks for data integrity. + /// If 0, verification is disabled (always use fast path when available). + /// If 1, verification is always enabled (always execute blocks). + verification_interval: u64, + /// Maximum blocks the startup check is willing to schedule for auto-prune. See + /// [`DEFAULT_MAX_PRUNE_BLOCKS_STARTUP`]. + max_prune_blocks_startup: u64, +} + +impl MorphProofsExEx +where + Node: FullNodeComponents, +{ + /// Create a new `MorphProofsExEx` instance. + pub fn new(ctx: ExExContext, storage: MorphProofsStorage) -> Self { + MorphProofsExExBuilder::new(ctx, storage).build() + } + + /// Create a new builder for `MorphProofsExEx`. + pub const fn builder( + ctx: ExExContext, + storage: MorphProofsStorage, + ) -> MorphProofsExExBuilder { + MorphProofsExExBuilder::new(ctx, storage) + } +} + +impl MorphProofsExEx +where + Node: FullNodeComponents>, + Primitives: NodePrimitives, + Storage: MorphProofsBatchStore + Clone + 'static, +{ + /// Main execution loop for the `ExEx` + pub async fn run(mut self) -> eyre::Result<()> { + self.ensure_initialized()?; + let sync_target = self.spawn_sync_task(); + + // If storage is behind tip, start syncing immediately rather than waiting + // for the first notification. + let best_block = self.ctx.provider().best_block_number()?; + let latest_stored = self + .storage + .get_latest_block_number()? + .map(|(n, _)| n) + .unwrap_or(0); + if latest_stored < best_block { + info!( + target: "morph::proofs_exex", + latest_stored, + best_block, + "Storage behind tip, starting sync immediately" + ); + sync_target.update_state(SyncTargetState::SyncUpTo { to: best_block }); + } + + let prune_task = MorphProofStoragePrunerTask::new( + self.storage.clone(), + self.ctx.provider().clone(), + self.proofs_history_window, + self.proofs_history_prune_interval, + ); + self.ctx + .task_executor() + .spawn_with_graceful_shutdown_signal(|signal| Box::pin(prune_task.run(signal))); + + self.ctx.notifications.set_without_head(); + + while let Some(notification) = self.ctx.notifications.try_next().await? { + self.handle_notification(notification, &sync_target)?; + } + + Ok(()) + } + + /// Ensure proofs storage is initialized + fn ensure_initialized(&self) -> eyre::Result<()> { + // Check if proofs storage is initialized + let earliest_block_number = match self.storage.get_earliest_block_number()? { + Some((n, _)) => n, + None => { + return Err(eyre::eyre!( + "Proofs storage not initialized. Please run 'morph-reth proofs init --proofs-history.storage-path ' first." + )); + } + }; + + let (latest_block_number, latest_block_hash) = match self + .storage + .get_latest_block_number()? + { + Some(latest) => latest, + None => { + return Err(eyre::eyre!( + "Proofs storage not initialized. Please run 'morph-reth proofs init --proofs-history.storage-path ' first." + )); + } + }; + + // Check if we have accumulated too much history for the configured window. + // If the gap between what we have and what we want to keep is too large, the auto-pruner + // will stall the node. + let target_earliest = latest_block_number.saturating_sub(self.proofs_history_window); + if target_earliest > earliest_block_number { + let blocks_to_prune = target_earliest - earliest_block_number; + if blocks_to_prune > self.max_prune_blocks_startup { + return Err(eyre::eyre!( + "Configuration requires pruning {} blocks, which exceeds the safety threshold of {}. \ + Huge prune operations can stall the node. \ + Please run 'morph-reth proofs prune' manually before starting the node.", + blocks_to_prune, + self.max_prune_blocks_startup + )); + } + } + + let canonical_hash = self + .ctx + .provider() + .block_hash(latest_block_number)? + .ok_or_else(|| eyre::eyre!( + "Proofs storage latest block {latest_block_number} is missing from the canonical chain" + ))?; + if canonical_hash != latest_block_hash { + return Err(eyre::eyre!( + "Proofs storage latest block hash mismatch at {latest_block_number}: stored {latest_block_hash}, canonical {canonical_hash}" + )); + } + + ProofHistoryMetrics::set_window(earliest_block_number, latest_block_number); + + Ok(()) + } + + /// Spawn the background sync task and return the shared [`SyncTarget`]. + /// + /// The sync target buffers trie data from notifications so the sync loop + /// can use pre-computed data for blocks even when it is many blocks behind + /// the chain tip. Blocks whose trie data was evicted from the cache fall + /// back to full execution. + fn spawn_sync_task(&self) -> Arc { + let sync_target = Arc::new(SyncTarget::new()); + let task_sync_target = Arc::clone(&sync_target); + + let task_storage = self.storage.clone(); + let task_provider = self.ctx.provider().clone(); + let task_evm_config = self.ctx.evm_config().clone(); + let verification_interval = self.verification_interval; + + self.ctx.task_executor().spawn_critical_task( + "morph::proofs_exex::proofs_storage_sync_loop", + async move { + let storage = task_storage.clone(); + let task_collector = + LiveTrieCollector::new(task_evm_config, task_provider.clone(), &storage); + Self::sync_loop( + task_sync_target, + task_storage, + task_provider, + &task_collector, + verification_interval, + ) + .await + .unwrap_or_else(|error| panic!("proof storage sync loop failed: {error:?}")); + }, + ); + + sync_target + } + + async fn sync_loop( + sync_target: Arc, + storage: MorphProofsStorage, + provider: Node::Provider, + collector: &LiveTrieCollector<'_, Node::Evm, Node::Provider, Storage>, + verification_interval: u64, + ) -> eyre::Result<()> { + info!(target: "morph::proofs_exex", "Starting proofs storage sync loop"); + + loop { + let Some(state) = sync_target.take_state() else { + sync_target.notified().await; + continue; + }; + + match state { + SyncTargetState::Revert { revert_to } => { + Self::handle_revert(&storage, collector, revert_to)?; + sync_target.mark_revert_complete(&revert_to); + } + SyncTargetState::RevertThenSync { revert_to, sync_to } => { + Self::handle_revert(&storage, collector, revert_to)?; + sync_target.mark_revert_complete(&revert_to); + Self::sync_forward( + &sync_target, + &storage, + &provider, + collector, + verification_interval, + sync_to, + ) + .await; + } + SyncTargetState::SyncUpTo { to } => { + Self::sync_forward( + &sync_target, + &storage, + &provider, + collector, + verification_interval, + to, + ) + .await; + } + } + } + } + + fn handle_revert( + storage: &MorphProofsStorage, + collector: &LiveTrieCollector<'_, Node::Evm, Node::Provider, Storage>, + revert_to: BlockWithParent, + ) -> eyre::Result<()> { + let latest = match storage.get_latest_block_number()? { + Some((n, _)) => n, + None => return Ok(()), + }; + + if latest >= revert_to.block.number { + info!( + target: "morph::proofs_exex", + revert_to = revert_to.block.number, + latest, + "Reverting proofs storage" + ); + collector.unwind_history(revert_to).map_err(|error| { + error!(target: "morph::proofs_exex", ?error, "Failed to revert proofs storage"); + error + })?; + } else { + debug!( + target: "morph::proofs_exex", + revert_to = revert_to.block.number, + latest, + "Revert target beyond stored blocks, skipping" + ); + } + Ok(()) + } + + #[cfg(test)] + fn handle_revert_for_test(&self, revert_to: BlockWithParent) -> eyre::Result<()> { + let collector = LiveTrieCollector::new( + self.ctx.evm_config().clone(), + self.ctx.provider().clone(), + &self.storage, + ); + Self::handle_revert(&self.storage, &collector, revert_to) + } + + async fn sync_forward( + sync_target: &SyncTarget, + storage: &MorphProofsStorage, + provider: &Node::Provider, + collector: &LiveTrieCollector<'_, Node::Evm, Node::Provider, Storage>, + verification_interval: u64, + target: u64, + ) { + loop { + // Check for higher-priority state (e.g. revert) before processing. + if sync_target.has_pending_state() { + return; + } + + let latest = match storage.get_latest_block_number() { + Ok(Some((n, _))) => n, + Ok(None) => { + error!(target: "morph::proofs_exex", "No blocks stored in proofs storage during sync"); + return; + } + Err(e) => { + error!(target: "morph::proofs_exex", error = ?e, "Failed to get latest block"); + return; + } + }; + + if latest >= target { + return; + } + + let end = latest + .saturating_add(SYNC_BLOCKS_PER_TURN as u64) + .min(target); + info!( + target: "morph::proofs_exex", + start = latest + 1, + end, + target, + blocks = end - latest, + "Processing proofs storage sync turn" + ); + + let mut batch: Vec> = + Vec::with_capacity((end - latest) as usize); + for block_num in (latest + 1)..=end { + let cached = sync_target.take(block_num); + match Self::build_batch_entry(block_num, cached, provider, verification_interval) { + Ok(entry) => batch.push(entry), + Err(e) => { + error!(target: "morph::proofs_exex", block_number = block_num, error = ?e, "Preparing block for batch failed"); + return; + } + } + } + + if let Err(e) = collector.execute_and_store_batch(batch) { + error!(target: "morph::proofs_exex", start = latest + 1, end, error = ?e, "Batch processing failed"); + return; + } + + info!(target: "morph::proofs_exex", latest_stored = latest, target, "Batch processed, yielding"); + task::yield_now().await; + } + } + + fn build_batch_entry( + block_number: u64, + cached: Option, + provider: &Node::Provider, + verification_interval: u64, + ) -> eyre::Result> { + let should_verify = + verification_interval > 0 && block_number.is_multiple_of(verification_interval); + let has_cached = cached.is_some(); + + if let Some(cached) = cached + && !should_verify + { + debug!( + target: "morph::proofs_exex", + block_number, + "Using pre-computed state from notification" + ); + return Ok(BatchBlock::Cached { + block_with_parent: cached.block_with_parent, + sorted_trie_updates: cached.trie_data.trie_updates(), + sorted_post_state: cached.trie_data.hashed_state(), + }); + } + + if has_cached { + info!( + target: "morph::proofs_exex", + block_number, + verification_interval, + "Periodic verification: performing full block execution despite cached data" + ); + } else { + debug!( + target: "morph::proofs_exex", + block_number, + "No cached trie data, falling back to full execution" + ); + } + + debug!( + target: "morph::proofs_exex", + block_number, + "Fetching block from provider for execution", + ); + + let block = provider + .recovered_block(block_number.into(), TransactionVariant::NoHash)? + .ok_or_else(|| eyre::eyre!("Missing block {} in provider", block_number))?; + + Ok(BatchBlock::Execute(Box::new(block))) + } + + fn handle_notification( + &self, + notification: ExExNotification, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + match ¬ification { + ExExNotification::ChainCommitted { new } => { + self.handle_chain_committed(Arc::clone(new), sync_target)? + } + ExExNotification::ChainReorged { old, new } => { + self.handle_chain_reorged(Arc::clone(old), Arc::clone(new), sync_target)? + } + ExExNotification::ChainReverted { old } => { + self.handle_chain_reverted(Arc::clone(old), sync_target)? + } + } + + if let Some(committed_chain) = notification.committed_chain() { + let tip = committed_chain.tip().num_hash(); + debug!( + target: "morph::proofs_exex", + block_number = tip.number, + block_hash = ?tip.hash, + "Sending FinishedHeight event" + ); + self.ctx.events.send(ExExEvent::FinishedHeight(tip))?; + } + + Ok(()) + } + + fn handle_chain_committed( + &self, + new: Arc>, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + debug!( + target: "morph::proofs_exex", + block_number = new.tip().number(), + block_hash = ?new.tip().hash(), + "ChainCommitted notification received", + ); + + // Cache trie data for all blocks in the chain so the sync loop can + // use pre-computed data even when it is many blocks behind. + let total_blocks = new.blocks().len(); + let mut cached_count = 0usize; + for (&block_number, block) in new.blocks() { + if let Some(trie_data) = new.trie_data_at(block_number) { + sync_target.insert( + block_number, + CachedBlockTrieData { + block_with_parent: block.block_with_parent(), + trie_data: trie_data.clone(), + }, + ); + cached_count += 1; + } else { + debug!( + target: "morph::proofs_exex", + block_number, + "Notification block missing trie data" + ); + } + } + + debug!( + target: "morph::proofs_exex", + tip = new.tip().number(), + total_blocks, + cached_count, + missing = total_blocks - cached_count, + "Cached notification trie data" + ); + + sync_target.update_state(SyncTargetState::SyncUpTo { + to: new.tip().number(), + }); + Ok(()) + } + + fn handle_chain_reorged( + &self, + old: Arc>, + new: Arc>, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + info!( + target: "morph::proofs_exex", + old_block_number = old.tip().number(), + old_block_hash = ?old.tip().hash(), + new_block_number = new.tip().number(), + new_block_hash = ?new.tip().hash(), + "ChainReorged notification received", + ); + + if old.fork_block() != new.fork_block() { + return Err(eyre::eyre!( + "Fork blocks do not match: old fork block {:?}, new fork block {:?}", + old.fork_block(), + new.fork_block() + )); + } + + let first_old = old.first().block_with_parent(); + + // Invalidate any cached blocks from the old chain. + sync_target.clear_from(first_old.block.number); + + // Cache trie data for all blocks in the new chain. + for (&block_number, block) in new.blocks() { + if let Some(trie_data) = new.trie_data_at(block_number) { + sync_target.insert( + block_number, + CachedBlockTrieData { + block_with_parent: block.block_with_parent(), + trie_data: trie_data.clone(), + }, + ); + } else { + debug!( + target: "morph::proofs_exex", + block_number, + "Reorged block missing trie data" + ); + } + } + + sync_target.update_state(SyncTargetState::RevertThenSync { + revert_to: first_old, + sync_to: new.tip().number(), + }); + + Ok(()) + } + + fn handle_chain_reverted( + &self, + old: Arc>, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + info!( + target: "morph::proofs_exex", + old_block_number = old.tip().number(), + old_block_hash = ?old.tip().hash(), + "ChainReverted notification received", + ); + + let first_old = old.first().block_with_parent(); + + // Invalidate any cached blocks that are being reverted. + sync_target.clear_from(first_old.block.number); + + sync_target.update_state(SyncTargetState::Revert { + revert_to: first_old, + }); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, default::Default, sync::Arc, time::Duration}; + + use alloy_consensus::private::alloy_primitives::B256; + use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; + use morph_proofs::{BlockStateDiff, MdbxProofsStorage, MorphProofsStorage, MorphProofsStore}; + use reth_db::test_utils::tempdir_path; + use reth_ethereum_primitives::{Block, Receipt}; + use reth_execution_types::{Chain, ExecutionOutcome}; + use reth_primitives_traits::RecoveredBlock; + use reth_trie::{ + ComputedTrieData, HashedPostStateSorted, LazyTrieData, updates::TrieUpdatesSorted, + }; + + use super::*; + + // ------------------------------------------------------------------------- + // Helpers: deterministic blocks and deterministic Chain with precomputed updates + // ------------------------------------------------------------------------- + fn b256(byte: u8) -> B256 { + B256::new([byte; 32]) + } + + // deterministic hash from block number: 0 -> 0x00.., 1 -> 0x01.., etc. + fn hash_for_num(num: u64) -> B256 { + // if you only care about small test numbers, this is enough: + b256(num as u8) + + // If you want to avoid wrapping when num > 255, use something like: + // let mut out = [0u8; 32]; + // out[0..8].copy_from_slice(&num.to_be_bytes()); + // B256::new(out) + } + + fn mk_block(num: u64) -> RecoveredBlock { + let mut b: RecoveredBlock = Default::default(); + b.set_block_number(num); + b.set_hash(hash_for_num(num)); + b.set_parent_hash(hash_for_num(num - 1)); + b + } + + fn mk_chain_with_updates( + from: u64, + to: u64, + hash_override: Option, + ) -> Chain { + let mut blocks: Vec> = Vec::new(); + let mut trie_data = BTreeMap::new(); + + for n in from..=to { + let mut b = mk_block(n); + if let Some(hash) = hash_override { + b.set_hash(hash); + } + blocks.push(b); + + let data = LazyTrieData::ready(ComputedTrieData::new( + Arc::new(HashedPostStateSorted::default()), + Arc::new(TrieUpdatesSorted::default()), + )); + trie_data.insert(n, data); + } + + let execution_outcome: ExecutionOutcome = ExecutionOutcome { + bundle: Default::default(), + receipts: Vec::new(), + requests: Vec::new(), + first_block: from, + }; + + Chain::new(blocks, execution_outcome, trie_data) + } + + /// Store blocks directly into proofs storage (bypasses the sync loop). + fn store_blocks(from: u64, to: u64, storage: &MorphProofsStorage) { + for n in from..=to { + let chain = mk_chain_with_updates(n, n, None); + let block = chain.blocks().get(&n).unwrap(); + storage + .store_trie_updates(block.block_with_parent(), BlockStateDiff::default()) + .expect("store trie update"); + } + } + + fn init_storage_at( + storage: MorphProofsStorage, + genesis_block: NumHash, + ) { + storage + .set_earliest_block_number(genesis_block.number, genesis_block.hash) + .expect("set earliest"); + storage + .store_trie_updates( + BlockWithParent::new(genesis_block.hash, genesis_block), + BlockStateDiff::default(), + ) + .expect("store trie update"); + } + + fn init_storage(storage: MorphProofsStorage) { + init_storage_at(storage, NumHash::new(0, b256(0x00))); + } + + // Initialize exex with config + fn build_test_exex( + ctx: ExExContext, + storage: MorphProofsStorage, + ) -> MorphProofsExEx + where + NodeT: FullNodeComponents, + Store: MorphProofsStore + Clone + 'static, + { + MorphProofsExEx::builder(ctx, storage) + .with_proofs_history_window(20) + .with_proofs_history_prune_interval(Duration::from_secs(3600)) + .with_verification_interval(1000) + .with_max_prune_blocks_startup(1000) + .build() + } + + #[tokio::test] + async fn handle_notification_chain_committed() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let new_chain = Arc::new(mk_chain_with_updates(1, 1, None)); + let notif = ExExNotification::ChainCommitted { new: new_chain }; + + let sync_target = SyncTarget::new(); + + exex.handle_notification(notif, &sync_target) + .expect("handle chain commit"); + + // Committed blocks are cached in sync target, not stored directly + assert!(sync_target.take(1).is_some(), "block 1 should be cached"); + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 1 })); + } + + #[tokio::test] + async fn handle_notification_chain_committed_caches_already_stored_blocks() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + // Pre-store blocks 1..5 so storage is at block 5 + store_blocks(1, 5, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Handle notification for block 5 which is already stored - still caches + let new_chain = Arc::new(mk_chain_with_updates(5, 5, Some(hash_for_num(10)))); + let notif = ExExNotification::ChainCommitted { new: new_chain }; + exex.handle_notification(notif, &sync_target) + .expect("handle chain commit"); + + // State is set (sync loop will see latest >= target and skip) + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 5 })); + + // Storage is unchanged (notification handler doesn't write to storage) + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok"); + assert_eq!(latest.0, 5); + assert_eq!(latest.1, hash_for_num(5)); + } + + #[tokio::test] + async fn handle_notification_chain_reorged() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 10, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 10, and we want to reorg from block 6..12 + let old_chain = Arc::new(mk_chain_with_updates(6, 10, None)); + let new_chain = Arc::new(mk_chain_with_updates(6, 12, None)); + + // Notification: chain reorged 6..12 + let notif = ExExNotification::ChainReorged { + new: new_chain, + old: old_chain, + }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain re-orged"); + + // Should have RevertThenSync state + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 12 } + if revert_to.block.number == 6 + )); + + // New chain blocks should be cached + for n in 6..=12 { + assert!(sync_target.take(n).is_some(), "block {n} should be cached"); + } + + // Storage unchanged (sync loop handles the actual revert) + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 10); + } + + #[tokio::test] + async fn handle_notification_chain_reorged_beyond_stored_blocks() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 10, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 10, and we want to reorg from block 12..15 + // Both chains share the same fork block (block 11) + let old_chain = Arc::new(mk_chain_with_updates(12, 15, None)); + let new_chain = Arc::new(mk_chain_with_updates(12, 20, None)); + + // Notification: chain reorged 12..20 + let notif = ExExNotification::ChainReorged { + new: new_chain, + old: old_chain, + }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain re-orged"); + + // State is set; sync loop will detect revert is beyond stored blocks + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 12 + )); + + // Storage unchanged + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 10); + } + + #[tokio::test] + async fn handle_notification_chain_reverted() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 10, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 10, and we want to revert from block 9..10 + let old_chain = Arc::new(mk_chain_with_updates(9, 10, None)); + + // Notification: chain reverted 9..10 + let notif = ExExNotification::ChainReverted { old: old_chain }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain reverted"); + + // Should have Revert state + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } + if revert_to.block.number == 9 + )); + + // Storage unchanged (sync loop handles the actual revert) + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 10); + } + + #[tokio::test] + async fn handle_notification_chain_reverted_beyond_stored_blocks() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 5, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 5, and we want to revert from block 9..10 + let old_chain = Arc::new(mk_chain_with_updates(9, 10, None)); + + // Notification: chain reverted 9..10 + let notif = ExExNotification::ChainReverted { old: old_chain }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain reverted"); + + // State is set; sync loop will detect revert is beyond stored blocks + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } + if revert_to.block.number == 9 + )); + + // Storage unchanged + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 5); + } + + #[tokio::test] + async fn handle_revert_propagates_unwind_failure() { + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + init_storage(proofs.clone()); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + let exex = build_test_exex(ctx, proofs.clone()); + + let initial_anchor = BlockWithParent::new(b256(0x00), NumHash::new(0, b256(0x00))); + let error = exex + .handle_revert_for_test(initial_anchor) + .expect_err("unwinding the initial anchor must fail"); + + assert!( + error.to_string().contains("earliest"), + "unexpected unwind error: {error:?}" + ); + assert_eq!( + proofs + .get_latest_block_number() + .expect("latest") + .expect("initialized") + .0, + 0, + "failed unwind must leave proof history unchanged" + ); + } + + #[tokio::test] + async fn ensure_initialized_errors_on_storage_not_initialized() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + let _ = exex.ensure_initialized().expect_err("should return error"); + } + + #[tokio::test] + async fn ensure_initialized_errors_when_prune_exceeds_threshold() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + for i in 1..1100 { + proofs + .store_trie_updates( + BlockWithParent::new( + hash_for_num(i - 1), + BlockNumHash::new(i, hash_for_num(i)), + ), + BlockStateDiff::default(), + ) + .expect("store trie update"); + } + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + let _ = exex.ensure_initialized().expect_err("should return error"); + } + + #[tokio::test] + async fn ensure_initialized_succeeds() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + let (ctx, handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + init_storage_at(proofs.clone(), handle.genesis.num_hash()); + + let exex = build_test_exex(ctx, proofs.clone()); + exex.ensure_initialized().expect("should not return error"); + } + + #[tokio::test] + async fn handle_notification_schedules_async_on_gap() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + // Notification: chain committed 5..10 (Blocks 1,2,3,4 are missing from storage) + let new_chain = Arc::new(mk_chain_with_updates(5, 10, None)); + let notif = ExExNotification::ChainCommitted { new: new_chain }; + + let sync_target = SyncTarget::new(); + + // Process notification + exex.handle_notification(notif, &sync_target) + .expect("handle chain commit should return ok immediately"); + + // Verify the sync target state was set + let state = sync_target.take_state().expect("should have pending state"); + assert!( + matches!(state, SyncTargetState::SyncUpTo { to: 10 }), + "Should have scheduled sync to block 10" + ); + + // Verify Main Thread did NOT process it + // Because we didn't spawn the actual worker thread in this test, storage should still be at + // 0. This proves the 'handle_notification' returned instantly without doing the + // heavy lifting. + let latest = proofs + .get_latest_block_number() + .expect("get") + .expect("ok") + .0; + assert_eq!( + latest, 0, + "Main thread should not have processed the blocks synchronously" + ); + } +} diff --git a/crates/proofs-exex/src/sync_target.rs b/crates/proofs-exex/src/sync_target.rs new file mode 100644 index 0000000..5a26b6f --- /dev/null +++ b/crates/proofs-exex/src/sync_target.rs @@ -0,0 +1,637 @@ +//! Sync target with trie data cache for the background sync loop. +//! +//! Buffers trie data from exex notifications so the sync loop can use +//! pre-computed data even when it is many blocks behind the chain tip. +//! Routes committed, reverted, and reorged notifications through a +//! [`SyncTargetState`] state machine so the sync loop is the single +//! writer to proofs storage. + +use std::{collections::BTreeMap, sync::Mutex}; + +use alloy_eips::eip1898::BlockWithParent; +use reth_trie::LazyTrieData; +use tokio::sync::Notify; +use tracing::debug; + +/// Maximum number of blocks to cache trie data for. +const CACHE_CAPACITY: usize = 1024; + +/// Cached trie data for a single block. +#[derive(Debug)] +pub struct CachedBlockTrieData { + /// The block identifier with its parent hash. + pub block_with_parent: BlockWithParent, + /// The lazy trie data (hashed state + trie updates). + pub trie_data: LazyTrieData, +} + +/// The state of the sync target, describing what the sync loop should do next. +#[derive(Debug)] +pub enum SyncTargetState { + /// Sync forward to a specific block number. + SyncUpTo { + /// The target block number to sync up to. + to: u64, + }, + /// Revert to a specific block, then sync forward to a target. + RevertThenSync { + /// The first block to remove (inclusive) during the revert. + revert_to: BlockWithParent, + /// The target block number to sync up to after reverting. + sync_to: u64, + }, + /// Revert to a specific block without syncing forward. + Revert { + /// The first block to remove (inclusive) during the revert. + revert_to: BlockWithParent, + }, +} + +impl SyncTargetState { + const fn apply_next(&mut self, new: Self) { + *self = match (&*self, new) { + // If we are just syncing to tip already, replace with the new state. + (Self::SyncUpTo { .. }, new) => new, + + // If the new state is a revert, replace with the new state. + (_, Self::Revert { revert_to }) => Self::Revert { revert_to }, + (_, Self::RevertThenSync { revert_to, sync_to }) => { + Self::RevertThenSync { revert_to, sync_to } + } + + // If we're currently reverting, replace the sync to value with the new + // state. + ( + Self::RevertThenSync { revert_to, .. } | Self::Revert { revert_to }, + Self::SyncUpTo { to }, + ) => Self::RevertThenSync { + revert_to: *revert_to, + sync_to: to, + }, + }; + } +} + +/// Sync target that buffers trie data from recent exex notifications. +/// +/// Routes all notification types (committed, reverted, reorged) through a +/// [`SyncTargetState`] state machine so the sync loop is the single writer +/// to proofs storage. Uses a [`Notify`] to wake the sync loop when new +/// state is available. +/// +/// Trie data is accumulated in a bounded [`BTreeMap`] so the sync loop +/// can still use pre-computed trie data for blocks from earlier notifications. +pub struct SyncTarget { + cache: Mutex>, + state: Mutex>, + notify: Notify, +} + +impl Default for SyncTarget { + fn default() -> Self { + Self::new() + } +} + +impl SyncTarget { + /// Create a new `SyncTarget` with no cached data and no pending state. + pub fn new() -> Self { + Self { + cache: Mutex::new(BTreeMap::new()), + state: Mutex::new(None), + notify: Notify::new(), + } + } + + /// Update the sync target state and wake the sync loop. + /// + /// If there is already a pending state, the new state is merged using + /// `SyncTargetState::apply_next`. + pub fn update_state(&self, new: SyncTargetState) { + let mut state = self.state.lock().expect("SyncTarget lock poisoned"); + match state.as_mut() { + Some(current) => current.apply_next(new), + None => *state = Some(new), + } + drop(state); + self.notify.notify_one(); + } + + /// Take the current pending state, leaving `None` in its place. + /// + /// Used by the sync loop to consume the next action to perform. + pub fn take_state(&self) -> Option { + self.state.lock().expect("SyncTarget lock poisoned").take() + } + + /// Notify the sync target that a revert has been processed up to the given block. + /// + /// Only strips the revert portion if the pending revert target is at or above + /// `reverted_to` (i.e. already covered by the completed revert). If a deeper + /// revert arrived while processing, the pending state is left unchanged. + /// + /// - Covered `RevertThenSync` → `SyncUpTo` (keeps the sync target) + /// - Covered `Revert` → `None` + /// - Deeper revert or other states → unchanged + pub fn mark_revert_complete(&self, reverted_to: &BlockWithParent) { + let mut state = self.state.lock().expect("SyncTarget lock poisoned"); + match &*state { + Some(SyncTargetState::RevertThenSync { revert_to, sync_to }) + if revert_to.block.number >= reverted_to.block.number => + { + *state = Some(SyncTargetState::SyncUpTo { to: *sync_to }); + } + Some(SyncTargetState::Revert { revert_to }) + if revert_to.block.number >= reverted_to.block.number => + { + *state = None; + } + _ => {} + } + } + + /// Check if there is a pending state without consuming it. + /// + /// Used by the sync loop to interrupt forward sync when a higher-priority + /// state (e.g. revert) arrives. + pub fn has_pending_state(&self) -> bool { + self.state + .lock() + .expect("SyncTarget lock poisoned") + .is_some() + } + + /// Wait for a state change notification. + /// + /// Returns immediately if a notification arrived since the last call. + pub async fn notified(&self) { + self.notify.notified().await; + } + + /// Insert cached trie data for a block. + /// + /// Evicts the oldest entries when the cache exceeds capacity. + pub fn insert(&self, block_number: u64, data: CachedBlockTrieData) { + let mut cache = self.cache.lock().expect("SyncTarget lock poisoned"); + cache.insert(block_number, data); + let mut evicted = 0u64; + while cache.len() > CACHE_CAPACITY { + cache.pop_first(); + evicted += 1; + } + if evicted > 0 { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + evicted, + "Cache full, evicted oldest entries" + ); + } + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + cached_blocks = cache.len(), + "Cached trie data for block" + ); + } + + /// Take cached trie data for a specific block, removing it from the cache. + pub fn take(&self, block_number: u64) -> Option { + let result = self + .cache + .lock() + .expect("SyncTarget lock poisoned") + .remove(&block_number); + if result.is_some() { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + "Cache hit: trie data found for block" + ); + } else { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + "Cache miss: no trie data for block, will re-execute" + ); + } + result + } + + /// Remove all cached entries at or above the given block number. + /// + /// Used when a revert or reorg invalidates cached blocks. + pub fn clear_from(&self, block_number: u64) { + let mut cache = self.cache.lock().expect("SyncTarget lock poisoned"); + let removed = cache.split_off(&block_number); + if !removed.is_empty() { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + cleared = removed.len(), + "Cleared cached entries from block onward" + ); + } + } +} + +impl std::fmt::Debug for SyncTarget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SyncTarget") + .field( + "has_pending_state", + &self + .state + .lock() + .expect("SyncTarget lock poisoned") + .is_some(), + ) + .field( + "cached_blocks", + &self.cache.lock().expect("SyncTarget lock poisoned").len(), + ) + .finish() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use alloy_consensus::private::alloy_primitives::B256; + use alloy_eips::{NumHash, eip1898::BlockWithParent}; + use reth_trie::{ + ComputedTrieData, HashedPostStateSorted, LazyTrieData, updates::TrieUpdatesSorted, + }; + + use super::*; + + fn b256(byte: u8) -> B256 { + B256::new([byte; 32]) + } + + fn block_with_parent(num: u64) -> BlockWithParent { + BlockWithParent::new( + b256(num.wrapping_sub(1) as u8), + NumHash::new(num, b256(num as u8)), + ) + } + + fn dummy_cached_data(num: u64) -> CachedBlockTrieData { + CachedBlockTrieData { + block_with_parent: block_with_parent(num), + trie_data: LazyTrieData::ready(ComputedTrieData::new( + Arc::new(HashedPostStateSorted::default()), + Arc::new(TrieUpdatesSorted::default()), + )), + } + } + + // ---- SyncTargetState::apply_next tests ---- + + #[test] + fn apply_next_sync_replaced_by_sync() { + let mut state = SyncTargetState::SyncUpTo { to: 10 }; + state.apply_next(SyncTargetState::SyncUpTo { to: 20 }); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 20 })); + } + + #[test] + fn apply_next_sync_replaced_by_revert() { + let mut state = SyncTargetState::SyncUpTo { to: 10 }; + let revert_to = block_with_parent(5); + state.apply_next(SyncTargetState::Revert { revert_to }); + assert!( + matches!(state, SyncTargetState::Revert { revert_to } if revert_to.block.number == 5) + ); + } + + #[test] + fn apply_next_sync_replaced_by_revert_then_sync() { + let mut state = SyncTargetState::SyncUpTo { to: 10 }; + let revert_to = block_with_parent(5); + state.apply_next(SyncTargetState::RevertThenSync { + revert_to, + sync_to: 20, + }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 5 + )); + } + + #[test] + fn apply_next_revert_then_sync_keeps_revert_on_new_sync() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::RevertThenSync { + revert_to, + sync_to: 10, + }; + state.apply_next(SyncTargetState::SyncUpTo { to: 30 }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 30 } + if revert_to.block.number == 5 + )); + } + + #[test] + fn apply_next_revert_then_sync_replaced_by_new_revert() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::RevertThenSync { + revert_to, + sync_to: 10, + }; + let new_revert = block_with_parent(3); + state.apply_next(SyncTargetState::Revert { + revert_to: new_revert, + }); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } if revert_to.block.number == 3 + )); + } + + #[test] + fn apply_next_revert_gains_sync_target() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::Revert { revert_to }; + state.apply_next(SyncTargetState::SyncUpTo { to: 15 }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 15 } + if revert_to.block.number == 5 + )); + } + + #[test] + fn apply_next_revert_replaced_by_revert_then_sync() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::Revert { revert_to }; + let new_revert = block_with_parent(3); + state.apply_next(SyncTargetState::RevertThenSync { + revert_to: new_revert, + sync_to: 20, + }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 3 + )); + } + + // ---- SyncTarget state management tests ---- + + #[test] + fn new_sync_target_has_no_state() { + let target = SyncTarget::new(); + assert!(!target.has_pending_state()); + assert!(target.take_state().is_none()); + } + + #[test] + fn update_state_sets_pending() { + let target = SyncTarget::new(); + target.update_state(SyncTargetState::SyncUpTo { to: 10 }); + assert!(target.has_pending_state()); + } + + #[test] + fn take_state_clears_pending() { + let target = SyncTarget::new(); + target.update_state(SyncTargetState::SyncUpTo { to: 10 }); + let state = target.take_state(); + assert!(matches!(state, Some(SyncTargetState::SyncUpTo { to: 10 }))); + assert!(!target.has_pending_state()); + } + + #[test] + fn update_state_merges_with_existing() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(5); + target.update_state(SyncTargetState::Revert { revert_to }); + target.update_state(SyncTargetState::SyncUpTo { to: 20 }); + + let state = target.take_state().expect("should have state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 5 + )); + } + + // ---- mark_revert_complete tests ---- + + #[test] + fn mark_revert_complete_clears_covered_revert() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(5); + target.update_state(SyncTargetState::Revert { revert_to }); + + target.mark_revert_complete(&block_with_parent(5)); + assert!(!target.has_pending_state()); + } + + #[test] + fn mark_revert_complete_strips_covered_revert_then_sync() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(5); + target.update_state(SyncTargetState::RevertThenSync { + revert_to, + sync_to: 20, + }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should have state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 20 })); + } + + #[test] + fn mark_revert_complete_keeps_deeper_revert() { + let target = SyncTarget::new(); + // A deeper revert (block 3) arrived while we were reverting to block 5 + let revert_to = block_with_parent(3); + target.update_state(SyncTargetState::Revert { revert_to }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should still have state"); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } if revert_to.block.number == 3 + )); + } + + #[test] + fn mark_revert_complete_keeps_deeper_revert_then_sync() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(3); + target.update_state(SyncTargetState::RevertThenSync { + revert_to, + sync_to: 20, + }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should still have state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 3 + )); + } + + #[test] + fn mark_revert_complete_clears_shallower_revert() { + let target = SyncTarget::new(); + // Pending revert to block 8 is shallower than what we reverted to (block 5) + let revert_to = block_with_parent(8); + target.update_state(SyncTargetState::Revert { revert_to }); + + target.mark_revert_complete(&block_with_parent(5)); + assert!(!target.has_pending_state()); + } + + #[test] + fn mark_revert_complete_noop_on_sync_up_to() { + let target = SyncTarget::new(); + target.update_state(SyncTargetState::SyncUpTo { to: 20 }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should still have state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 20 })); + } + + #[test] + fn mark_revert_complete_noop_on_empty() { + let target = SyncTarget::new(); + target.mark_revert_complete(&block_with_parent(5)); + assert!(!target.has_pending_state()); + } + + // ---- Cache tests ---- + + #[test] + fn insert_and_take_cached_data() { + let target = SyncTarget::new(); + target.insert(42, dummy_cached_data(42)); + + let taken = target.take(42); + assert!(taken.is_some()); + assert_eq!(taken.unwrap().block_with_parent.block.number, 42); + + // Second take returns None (was removed) + assert!(target.take(42).is_none()); + } + + #[test] + fn take_nonexistent_block_returns_none() { + let target = SyncTarget::new(); + assert!(target.take(999).is_none()); + } + + #[test] + fn clear_from_removes_blocks_at_and_above() { + let target = SyncTarget::new(); + for n in 1..=10 { + target.insert(n, dummy_cached_data(n)); + } + + target.clear_from(6); + + // Blocks 1..=5 remain + for n in 1..=5 { + assert!(target.take(n).is_some(), "block {n} should still be cached"); + } + // Blocks 6..=10 removed + for n in 6..=10 { + assert!( + target.take(n).is_none(), + "block {n} should have been cleared" + ); + } + } + + #[test] + fn clear_from_at_start_clears_everything() { + let target = SyncTarget::new(); + for n in 5..=10 { + target.insert(n, dummy_cached_data(n)); + } + target.clear_from(5); + for n in 5..=10 { + assert!(target.take(n).is_none()); + } + } + + #[test] + fn clear_from_beyond_cache_is_noop() { + let target = SyncTarget::new(); + for n in 1..=5 { + target.insert(n, dummy_cached_data(n)); + } + target.clear_from(100); + for n in 1..=5 { + assert!(target.take(n).is_some()); + } + } + + #[test] + fn cache_evicts_oldest_when_full() { + let target = SyncTarget::new(); + + // Fill to capacity + 10 + for n in 1..=(CACHE_CAPACITY as u64 + 10) { + target.insert(n, dummy_cached_data(n)); + } + + // Oldest 10 blocks should have been evicted + for n in 1..=10 { + assert!( + target.take(n).is_none(), + "block {n} should have been evicted" + ); + } + + // Block 11 onward should still be present + assert!(target.take(11).is_some()); + assert!(target.take(CACHE_CAPACITY as u64 + 10).is_some()); + } + + #[test] + fn default_creates_empty_sync_target() { + let target = SyncTarget::default(); + assert!(!target.has_pending_state()); + assert!(target.take(0).is_none()); + } + + #[test] + fn debug_impl_does_not_panic() { + let target = SyncTarget::new(); + target.insert(1, dummy_cached_data(1)); + target.update_state(SyncTargetState::SyncUpTo { to: 5 }); + let debug_str = format!("{target:?}"); + assert!(debug_str.contains("SyncTarget")); + assert!(debug_str.contains("has_pending_state")); + assert!(debug_str.contains("cached_blocks")); + } + + #[tokio::test] + async fn notified_wakes_on_update_state() { + let target = Arc::new(SyncTarget::new()); + let target_clone = Arc::clone(&target); + + // Spawn a task that waits for notification + let handle = tokio::spawn(async move { + target_clone.notified().await; + target_clone.take_state() + }); + + // Give the task time to start waiting + tokio::task::yield_now().await; + + target.update_state(SyncTargetState::SyncUpTo { to: 42 }); + + let result = handle.await.expect("task should complete"); + assert!(matches!(result, Some(SyncTargetState::SyncUpTo { to: 42 }))); + } +} diff --git a/crates/proofs/Cargo.toml b/crates/proofs/Cargo.toml new file mode 100644 index 0000000..db4b160 --- /dev/null +++ b/crates/proofs/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "morph-proofs" +description = "Bounded historical MPT storage for Morph eth_getProof" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +alloy-eips.workspace = true +alloy-primitives.workspace = true +auto_impl.workspace = true +bincode = { workspace = true, features = ["serde"] } +bytes.workspace = true +derive_more.workspace = true +metrics.workspace = true +parking_lot.workspace = true +serde.workspace = true +strum = { workspace = true, features = ["derive"] } +thiserror.workspace = true +tokio = { workspace = true, features = ["sync"] } +tracing.workspace = true + +reth-codecs.workspace = true +reth-db = { workspace = true, features = ["mdbx"] } +reth-evm.workspace = true +reth-execution-errors.workspace = true +reth-metrics.workspace = true +reth-primitives-traits.workspace = true +reth-provider.workspace = true +reth-revm = { workspace = true, features = ["witness"] } +reth-tasks.workspace = true +reth-trie = { workspace = true, features = ["serde"] } +reth-trie-common = { workspace = true, features = ["serde"] } + +[dev-dependencies] +alloy-primitives = { workspace = true, features = ["rand"] } +mockall.workspace = true +reth-db = { workspace = true, features = ["test-utils"] } +reth-db-api = { workspace = true, features = ["test-utils"] } +reth-storage-errors.workspace = true +tempfile.workspace = true + +[features] +default = [] +test-utils = [] diff --git a/crates/proofs/NOTICE.md b/crates/proofs/NOTICE.md new file mode 100644 index 0000000..f60916e --- /dev/null +++ b/crates/proofs/NOTICE.md @@ -0,0 +1,32 @@ +# Attribution + +The versioned trie storage and proof-history algorithms in this crate are adapted from +the Base repository at commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88`: + + + +The adapted code is used under Base's MIT license. Morph-specific changes include an +MDBX-only surface, durable chain-identity metadata, and integration with Morph's node, +RPC, and lifecycle rules. + +## Base MIT License + +Copyright (c) 2025 Base + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/proofs/src/api.rs b/crates/proofs/src/api.rs new file mode 100644 index 0000000..0d01409 --- /dev/null +++ b/crates/proofs/src/api.rs @@ -0,0 +1,435 @@ +//! Storage API for external storage of intermediary trie nodes. + +use std::{fmt::Debug, time::Duration}; + +use alloy_eips::{BlockNumHash, eip1898::BlockWithParent}; +use alloy_primitives::{B256, U256}; +use auto_impl::auto_impl; +use derive_more::{AddAssign, Constructor}; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{ + BranchNodeCompact, HashedPostStateSorted, Nibbles, StoredNibbles, updates::TrieUpdatesSorted, +}; + +use crate::{ + MorphProofsStorageResult, + db::{HashedStorageKey, StorageTrieKey}, +}; + +/// Type alias for bulk storage branch entries: a list of (hashed address, storage nodes) pairs. +pub type StorageBranchEntries = Vec<(B256, Vec<(Nibbles, Option)>)>; + +/// Diff of trie updates and post state for a block. +#[derive(Debug, Clone, Default)] +pub struct BlockStateDiff { + /// Trie updates for branch nodes + pub sorted_trie_updates: TrieUpdatesSorted, + /// Post state for leaf nodes (accounts and storage) + pub sorted_post_state: HashedPostStateSorted, +} + +impl BlockStateDiff { + /// Extend the [` BlockStateDiff`] from other latest [`BlockStateDiff`] + pub fn extend_ref(&mut self, other: &Self) { + self.sorted_trie_updates + .extend_ref_and_sort(&other.sorted_trie_updates); + self.sorted_post_state + .extend_ref_and_sort(&other.sorted_post_state); + } +} + +/// Counts of trie updates written to storage. +#[derive(Debug, Clone, Default, AddAssign, Constructor, Eq, PartialEq)] +pub struct WriteCounts { + /// Number of account trie updates written + pub account_trie_updates_written_total: u64, + /// Number of storage trie updates written + pub storage_trie_updates_written_total: u64, + /// Number of hashed accounts written + pub hashed_accounts_written_total: u64, + /// Number of hashed storages written + pub hashed_storages_written_total: u64, +} + +/// Duration metrics for block processing. +#[derive(Debug, Default, Clone)] +pub struct OperationDurations { + /// Total time to process a block (end-to-end) in seconds + pub total_duration_seconds: Duration, + /// Time spent executing the block (EVM) in seconds + pub execution_duration_seconds: Duration, + /// Time spent calculating state root in seconds + pub state_root_duration_seconds: Duration, + /// Time spent writing trie updates to storage in seconds + pub write_duration_seconds: Duration, +} + +/// Trait for reading trie nodes from the database. +/// +/// Only leaf nodes and some branch nodes are stored. The bottom layer of branch nodes +/// are not stored to reduce write amplification. This matches Reth's non-historical trie storage. +#[auto_impl(Arc)] +pub trait MorphProofsStore: Send + Sync + Debug { + /// Cursor for iterating over trie branches. + type StorageTrieCursor<'tx>: TrieStorageCursor + 'tx + where + Self: 'tx; + + /// Cursor for iterating over account trie branches. + type AccountTrieCursor<'tx>: TrieCursor + 'tx + where + Self: 'tx; + + /// Cursor for iterating over storage leaves. + type StorageCursor<'tx>: HashedStorageCursor + Send + Sync + 'tx + where + Self: 'tx; + + /// Cursor for iterating over account leaves. + type AccountHashedCursor<'tx>: HashedCursor + Send + Sync + 'tx + where + Self: 'tx; + + /// Transaction type used by request-scoped cursor factories. + type Tx<'tx>: Debug + Send + Sync + where + Self: 'tx; + + /// Get the earliest block number and hash that has been stored + /// + /// This is used to determine the block number of trie nodes with block number 0. + /// All earliest block numbers are stored in 0 to reduce updates required to prune trie nodes. + fn get_earliest_block_number(&self) -> MorphProofsStorageResult>; + + /// Get the latest block number and hash that has been stored + fn get_latest_block_number(&self) -> MorphProofsStorageResult>; + + /// Get the earliest block number and hash visible to an existing read transaction. + fn get_earliest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Get the latest block number and hash visible to an existing read transaction. + fn get_latest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Get a trie cursor for the storage backend + fn storage_trie_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Get a trie cursor for the account backend + fn account_trie_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Get a storage cursor for the storage backend + fn storage_hashed_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Get an account hashed cursor for the storage backend + fn account_hashed_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Open a transaction for a request-scoped cursor factory. + fn ro_tx<'tx>(&'tx self) -> MorphProofsStorageResult>; + + /// Storage trie cursor reusing `tx`. + fn storage_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Account trie cursor reusing `tx`. + fn account_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Storage hashed cursor reusing `tx`. + fn storage_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Account hashed cursor reusing `tx`. + fn account_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Store a batch of trie updates. + /// + /// If wiped is true, the entire storage trie is wiped, but this is unsupported going forward, + /// so should only happen for legacy reasons. + fn store_trie_updates( + &self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult; + + /// Fetch all updates for a given block number. + fn fetch_trie_updates(&self, block_number: u64) -> MorphProofsStorageResult; + + /// Applies [`BlockStateDiff`] to the earliest state (updating/deleting nodes) and updates the + /// earliest block number. + fn prune_earliest_state( + &self, + new_earliest_block_ref: BlockWithParent, + ) -> MorphProofsStorageResult; + + /// Remove account, storage and trie updates from historical storage for all blocks till + /// the specified block (inclusive). + fn unwind_history(&self, to: BlockWithParent) -> MorphProofsStorageResult<()>; + + /// Deletes all updates > `latest_common_block` and replaces them with the new updates. + fn replace_updates( + &self, + latest_common_block: BlockNumHash, + blocks_to_add: Vec<(BlockWithParent, BlockStateDiff)>, + ) -> MorphProofsStorageResult<()>; + + /// Set the earliest block number and hash that has been stored + fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()>; +} + +/// Session-scoped store of trie updates that amortizes write overhead across multiple +/// block writes within a single batch. +/// +/// Reads through the session observe writes from earlier [`MorphProofsBatchSession::store_trie_updates`] +/// calls in the same session, enabling cold catch-up where `block N+1` is computed against +/// `block N` written earlier in the same session. +/// +/// All cursor methods mirror [`MorphProofsStore`] but read through the session's active view. +#[auto_impl(&mut)] +pub trait MorphProofsBatchSession: Send + Sync + Debug { + /// Cursor for iterating over storage trie branches in the active session. + type StorageTrieCursor<'a>: TrieStorageCursor + 'a + where + Self: 'a; + + /// Cursor for iterating over account trie branches in the active session. + type AccountTrieCursor<'a>: TrieCursor + 'a + where + Self: 'a; + + /// Cursor for iterating over storage leaves in the active session. + type StorageCursor<'a>: HashedStorageCursor + Send + Sync + 'a + where + Self: 'a; + + /// Cursor for iterating over account leaves in the active session. + type AccountHashedCursor<'a>: HashedCursor + Send + Sync + 'a + where + Self: 'a; + + /// Earliest stored block number/hash visible to the active transaction. + fn get_earliest_block_number(&self) -> MorphProofsStorageResult>; + + /// Latest stored block number/hash visible to this session (including writes from + /// earlier [`Self::store_trie_updates`] calls in this session). + fn get_latest_block_number(&self) -> MorphProofsStorageResult>; + + /// Storage trie cursor reading through the active transaction. + fn storage_trie_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Account trie cursor reading through the active transaction. + fn account_trie_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Storage hashed cursor reading through the active transaction. + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Account hashed cursor reading through the active transaction. + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Append-only write of `block_state_diff` for `block_ref` to the active session. + /// Subsequent reads through this session observe the new state immediately. + /// Durability semantics are backend-dependent; see [`MorphProofsBatchStore`]. + fn store_trie_updates( + &mut self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult; +} + +/// Storage that can open a [`MorphProofsBatchSession`] for amortized multi-block writes. +/// +/// MDBX commits the session atomically on `Ok` and rolls it back on `Err`. +/// The in-memory implementation is a test double and does not provide rollback. +pub trait MorphProofsBatchStore: MorphProofsStore { + /// Session type for this backend. + type BatchSession<'a>: MorphProofsBatchSession + 'a + where + Self: 'a; + + /// Run `f` inside one batch session. Atomicity and error-recovery semantics are + /// backend-dependent; see the trait-level documentation. + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult; +} + +impl MorphProofsBatchStore for std::sync::Arc { + type BatchSession<'a> + = T::BatchSession<'a> + where + Self: 'a; + + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult, + { + (**self).with_batch_session(f) + } +} + +/// Status of the initial state anchor. +#[derive(Debug, Clone, Copy, Default)] +pub enum InitialStateStatus { + /// Init isn't yet started + #[default] + NotStarted, + /// Init is in progress (some tables may already be populated) + InProgress, + /// Init completed successfully (all tables done + earliest block set) + Completed, +} + +/// Anchor for the initial state. +#[derive(Debug, Clone, Default)] +pub struct InitialStateAnchor { + /// The block for which the initial state is being initialized. None if initialization is not + /// yet started. + pub block: Option, + /// Whether initialization is still running or completed. + pub status: InitialStateStatus, + /// The latest key stored for `AccountTrieHistory`. + pub latest_account_trie_key: Option, + /// The latest key stored for `StorageTrieHistory`. + pub latest_storage_trie_key: Option, + /// The latest key stored for `HashedAccountHistory`. + pub latest_hashed_account_key: Option, + /// The latest key stored for `HashedStorageHistory`. + pub latest_hashed_storage_key: Option, +} + +/// Trait for storing and retrieving the initial state anchor. +#[auto_impl(Arc)] +pub trait MorphProofsInitialStateStore: Send + Sync + Debug { + /// Read the current anchor. + fn initial_state_anchor(&self) -> MorphProofsStorageResult; + + /// Create the anchor if it doesn't exist. + /// Returns `Err` if an anchor already exists (prevents accidental overwrite). + fn set_initial_state_anchor(&self, anchor: BlockNumHash) -> MorphProofsStorageResult<()>; + + /// Store a batch of account trie branches. Used for saving existing state. For live state + /// capture, use [store_trie_updates](MorphProofsStore::store_trie_updates). + fn store_account_branches( + &self, + account_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()>; + + /// Store a batch of storage trie branches. Used for saving existing state. + fn store_storage_branches( + &self, + hashed_address: B256, + storage_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()>; + + /// Store storage trie branches for multiple addresses in one initialization batch. + fn store_storage_branches_bulk( + &self, + entries: StorageBranchEntries, + ) -> MorphProofsStorageResult<()> { + for (hashed_address, nodes) in entries { + self.store_storage_branches(hashed_address, nodes)?; + } + Ok(()) + } + + /// Store a batch of account trie leaf nodes. Used for saving existing state. + fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> MorphProofsStorageResult<()>; + + /// Store a batch of storage trie leaf nodes. Used for saving existing state. + fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> MorphProofsStorageResult<()>; + + /// Store hashed storage slots for multiple addresses in one initialization batch. + fn store_hashed_storages_bulk( + &self, + entries: Vec<(B256, Vec<(B256, U256)>)>, + ) -> MorphProofsStorageResult<()> { + for (hashed_address, storages) in entries { + self.store_hashed_storages(hashed_address, storages)?; + } + Ok(()) + } + + /// Commit the initial state - mark the anchor as completed and also set the earliest block + /// number to anchor. + fn commit_initial_state(&self) -> MorphProofsStorageResult; +} diff --git a/crates/proofs/src/batch_provider.rs b/crates/proofs/src/batch_provider.rs new file mode 100644 index 0000000..11b67e0 --- /dev/null +++ b/crates/proofs/src/batch_provider.rs @@ -0,0 +1,313 @@ +//! State provider for an active [`MorphProofsBatchSession`] enabling reads to observe +//! uncommitted writes performed earlier in the same session. + +use std::fmt::Debug; + +use alloy_primitives::{ + keccak256, + map::{B256Map, HashMap}, +}; +use derive_more::Constructor; +use reth_primitives_traits::{Account, Bytecode}; +use reth_provider::{ + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, ProviderError, + ProviderResult, StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, +}; +use reth_revm::{ + db::BundleState, + primitives::{Address, B256, Bytes, StorageValue, alloy_primitives::BlockNumber}, +}; +use reth_trie::{ + StateRoot, StorageRoot, TrieType, + hashed_cursor::{HashedCursor, HashedPostStateCursorFactory}, + metrics::TrieRootMetrics, + proof, + trie_cursor::InMemoryTrieCursorFactory, + witness::TrieWitness, +}; +use reth_trie_common::{ + AccountProof, ExecutionWitnessMode, HashedPostState, HashedPostStateSorted, HashedStorage, + KeccakKeyHasher, MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::TrieUpdates, +}; + +use crate::{ + MorphProofsBatchHashedAccountCursorFactory, MorphProofsBatchTrieCursorFactory, + api::MorphProofsBatchSession, +}; + +/// State provider that reads through an active [`MorphProofsBatchSession`]'s transaction. +#[derive(Constructor)] +pub struct MorphProofsBatchStateProviderRef<'a, S: MorphProofsBatchSession> { + latest: Box, + session: &'a S, + block_number: BlockNumber, +} + +impl Debug for MorphProofsBatchStateProviderRef<'_, S> +where + S: MorphProofsBatchSession, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MorphProofsBatchStateProviderRef") + .field("session", &self.session) + .field("block_number", &self.block_number) + .finish() + } +} + +impl<'a, S: MorphProofsBatchSession> MorphProofsBatchStateProviderRef<'a, S> { + const fn factories( + &self, + ) -> ( + MorphProofsBatchTrieCursorFactory<'a, S>, + MorphProofsBatchHashedAccountCursorFactory<'a, S>, + ) { + ( + MorphProofsBatchTrieCursorFactory::new(self.session, self.block_number), + MorphProofsBatchHashedAccountCursorFactory::new(self.session, self.block_number), + ) + } +} + +impl BlockHashReader for MorphProofsBatchStateProviderRef<'_, S> { + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + self.latest.block_hash(number) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + self.latest.canonical_hashes_range(start, end) + } +} + +impl StateRootProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn state_root(&self, state: HashedPostState) -> ProviderResult { + let prefix_sets = state.construct_prefix_sets().freeze(); + let state_sorted = state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root() + .map_err(ProviderError::from) + } + + fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root() + .map_err(ProviderError::from) + } + + fn state_root_with_updates( + &self, + state: HashedPostState, + ) -> ProviderResult<(B256, TrieUpdates)> { + let prefix_sets = state.construct_prefix_sets().freeze(); + let state_sorted = state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root_with_updates() + .map_err(ProviderError::from) + } + + fn state_root_from_nodes_with_updates( + &self, + input: TrieInput, + ) -> ProviderResult<(B256, TrieUpdates)> { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root_with_updates() + .map_err(ProviderError::from) + } +} + +impl StorageRootProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + let prefix_set = storage.construct_prefix_set().freeze(); + let state_sorted = + HashedPostState::from_hashed_storage(keccak256(address), storage).into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StorageRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + address, + prefix_set, + TrieRootMetrics::new(TrieType::Custom("morph_historical_proofs_storage_batch")), + ) + .root() + .map_err(|err| ProviderError::Database(err.into())) + } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + let hashed_address = keccak256(address); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let (trie_factory, hashed_factory) = self.factories(); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_proof(slot) + .map_err(ProviderError::from) + } + + fn storage_multiproof( + &self, + address: Address, + slots: &[B256], + hashed_storage: HashedStorage, + ) -> ProviderResult { + let hashed_address = keccak256(address); + let targets = slots.iter().map(keccak256).collect(); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let (trie_factory, hashed_factory) = self.factories(); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_multiproof(targets) + .map_err(ProviderError::from) + } +} + +impl StateProofProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn proof( + &self, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> ProviderResult { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + proof::Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .account_proof(address, slots) + .map_err(ProviderError::from) + } + + fn multiproof( + &self, + input: TrieInput, + targets: MultiProofTargets, + ) -> ProviderResult { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + proof::Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .multiproof(targets) + .map_err(ProviderError::from) + } + + fn witness( + &self, + input: TrieInput, + target: HashedPostState, + _mode: ExecutionWitnessMode, + ) -> ProviderResult> { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + let result: B256Map = TrieWitness::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .always_include_root_node() + .compute(target) + .map_err(ProviderError::from)?; + Ok(result.into_values().collect()) + } +} + +impl HashedPostStateProvider + for MorphProofsBatchStateProviderRef<'_, S> +{ + fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { + HashedPostState::from_bundle_state::(bundle_state.state()) + } +} + +impl AccountReader for MorphProofsBatchStateProviderRef<'_, S> { + fn basic_account(&self, address: &Address) -> ProviderResult> { + let hashed_key = keccak256(address.0); + Ok(self + .session + .account_hashed_cursor(self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, account)| (key == hashed_key).then_some(account))) + } +} + +impl StateProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn storage(&self, address: Address, storage_key: B256) -> ProviderResult> { + let hashed_key = keccak256(storage_key); + Ok(self + .session + .storage_hashed_cursor(keccak256(address.0), self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, storage)| (key == hashed_key).then_some(storage))) + } +} + +impl BytecodeReader for MorphProofsBatchStateProviderRef<'_, S> { + fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { + self.latest.bytecode_by_hash(code_hash) + } +} diff --git a/crates/proofs/src/cursor.rs b/crates/proofs/src/cursor.rs new file mode 100644 index 0000000..ec04141 --- /dev/null +++ b/crates/proofs/src/cursor.rs @@ -0,0 +1,129 @@ +//! Implementation of [`HashedCursor`] and [`TrieCursor`] for +//! [`MorphProofsStorage`](crate::MorphProofsStorage). + +use alloy_primitives::{B256, U256}; +use derive_more::Constructor; +use reth_db::DatabaseError; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{BranchNodeCompact, Nibbles}; + +/// Manages reading storage or account trie nodes from [`TrieCursor`]. +#[derive(Debug, Clone, Constructor)] +pub struct MorphProofsTrieCursor(pub C); + +impl TrieCursor for MorphProofsTrieCursor +where + C: TrieCursor, +{ + #[inline] + fn seek_exact( + &mut self, + key: Nibbles, + ) -> Result, DatabaseError> { + self.0.seek_exact(key) + } + + #[inline] + fn seek( + &mut self, + key: Nibbles, + ) -> Result, DatabaseError> { + self.0.seek(key) + } + + #[inline] + fn next(&mut self) -> Result, DatabaseError> { + self.0.next() + } + + #[inline] + fn current(&mut self) -> Result, DatabaseError> { + self.0.current() + } + + #[inline] + fn reset(&mut self) { + self.0.reset() + } +} + +impl TrieStorageCursor for MorphProofsTrieCursor +where + C: TrieStorageCursor, +{ + #[inline] + fn set_hashed_address(&mut self, hashed_address: B256) { + self.0.set_hashed_address(hashed_address) + } +} + +/// Manages reading hashed account nodes from external storage. +#[derive(Debug, Clone, Constructor)] +pub struct MorphProofsHashedAccountCursor(pub C); + +impl HashedCursor for MorphProofsHashedAccountCursor +where + C: HashedCursor + Send + Sync, +{ + type Value = Account; + + #[inline] + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + self.0.seek(key) + } + + #[inline] + fn next(&mut self) -> Result, DatabaseError> { + self.0.next() + } + + #[inline] + fn reset(&mut self) { + self.0.reset() + } +} + +/// Manages reading hashed storage nodes from external storage. +#[derive(Debug, Clone, Constructor)] +pub struct MorphProofsHashedStorageCursor(pub C); + +impl HashedCursor for MorphProofsHashedStorageCursor +where + C: HashedCursor + Send + Sync, +{ + type Value = U256; + + #[inline] + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + self.0.seek(key) + } + + #[inline] + fn next(&mut self) -> Result, DatabaseError> { + self.0.next() + } + + #[inline] + fn reset(&mut self) { + self.0.reset() + } +} + +impl HashedStorageCursor for MorphProofsHashedStorageCursor +where + C: HashedStorageCursor + Send + Sync, +{ + #[inline] + fn is_storage_empty(&mut self) -> Result { + self.0.is_storage_empty() + } + + #[inline] + fn set_hashed_address(&mut self, hashed_address: B256) { + self.0.set_hashed_address(hashed_address) + } +} diff --git a/crates/proofs/src/cursor_factory.rs b/crates/proofs/src/cursor_factory.rs new file mode 100644 index 0000000..fa2dfcb --- /dev/null +++ b/crates/proofs/src/cursor_factory.rs @@ -0,0 +1,260 @@ +//! Implements [`TrieCursorFactory`] and [`HashedCursorFactory`] for [`MorphProofsStore`] types. +//! +//! Both factories borrow one read-only [`MorphProofsStore::Tx`] for their entire lifetime +//! and route every cursor allocation through the `*_with_tx` fast path. This mirrors +//! reth's own `DatabaseTrieCursorFactory` / `DatabaseHashedCursorFactory` pattern and is +//! what lets proof, state-root, and witness requests acquire exactly one MDBX +//! transaction. + +use alloy_primitives::B256; +use reth_db::DatabaseError; +use reth_trie::{hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory}; + +use crate::{ + MorphProofsHashedAccountCursor, MorphProofsHashedStorageCursor, MorphProofsStorage, + MorphProofsStore, MorphProofsTrieCursor, + api::MorphProofsBatchSession, + cursor::{ + MorphProofsHashedAccountCursor as RawHashedAccountCursor, + MorphProofsHashedStorageCursor as RawHashedStorageCursor, + MorphProofsTrieCursor as RawTrieCursor, + }, +}; + +/// Request-scoped factory that opens trie cursors against a shared read-only transaction. +/// +/// Holds a borrow of the transaction so every cursor allocation reuses the same MDBX +/// reader slot. See [`MorphProofsStore::Tx`] for the underlying contention story. +#[derive(Debug, Clone)] +pub struct MorphProofsTrieCursorFactory<'tx, 'db, S: MorphProofsStore> { + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, +} + +impl<'tx, 'db, S: MorphProofsStore> MorphProofsTrieCursorFactory<'tx, 'db, S> { + /// Initializes a request-scoped trie cursor factory bound to `tx`. + pub const fn new( + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, + ) -> Self { + Self { + storage, + tx, + block_number, + } + } +} + +impl<'tx, 'db, S> TrieCursorFactory for MorphProofsTrieCursorFactory<'tx, 'db, S> +where + for<'a> S: MorphProofsStore + 'db, + 'db: 'tx, +{ + type AccountTrieCursor<'a> + = MorphProofsTrieCursor> + where + Self: 'a; + type StorageTrieCursor<'a> + = MorphProofsTrieCursor> + where + Self: 'a; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { + Ok(MorphProofsTrieCursor::new( + self.storage + .account_trie_cursor_with_tx(self.tx, self.block_number) + .map_err(Into::::into)?, + )) + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(MorphProofsTrieCursor::new( + self.storage + .storage_trie_cursor_with_tx(self.tx, hashed_address, self.block_number) + .map_err(Into::::into)?, + )) + } +} + +/// Request-scoped factory that opens hashed cursors against a shared read-only transaction. +/// +/// Mirror of [`MorphProofsTrieCursorFactory`] for the hashed account/storage tries. +#[derive(Debug, Clone)] +pub struct MorphProofsHashedAccountCursorFactory<'tx, 'db, S: MorphProofsStore> { + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, +} + +impl<'tx, 'db, S: MorphProofsStore> MorphProofsHashedAccountCursorFactory<'tx, 'db, S> { + /// Initializes a request-scoped hashed cursor factory bound to `tx`. + pub const fn new( + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, + ) -> Self { + Self { + storage, + tx, + block_number, + } + } +} + +impl<'tx, 'db, S> HashedCursorFactory for MorphProofsHashedAccountCursorFactory<'tx, 'db, S> +where + for<'a> S: MorphProofsStore + 'db, + 'db: 'tx, +{ + type AccountCursor<'a> + = MorphProofsHashedAccountCursor> + where + Self: 'a; + type StorageCursor<'a> + = MorphProofsHashedStorageCursor> + where + Self: 'a; + + fn hashed_account_cursor(&self) -> Result, DatabaseError> { + Ok(MorphProofsHashedAccountCursor::new( + self.storage + .account_hashed_cursor_with_tx(self.tx, self.block_number)?, + )) + } + + fn hashed_storage_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(MorphProofsHashedStorageCursor::new( + self.storage.storage_hashed_cursor_with_tx( + self.tx, + hashed_address, + self.block_number, + )?, + )) + } +} + +/// Session-scoped trie cursor factory backed by a [`MorphProofsBatchSession`]. +/// +/// Cursors read from the session's active transaction and therefore observe writes +/// from earlier `store_trie_updates` calls in the same session. +#[derive(Debug)] +pub struct MorphProofsBatchTrieCursorFactory<'a, S: MorphProofsBatchSession> { + session: &'a S, + block_number: u64, +} + +impl Clone for MorphProofsBatchTrieCursorFactory<'_, S> { + fn clone(&self) -> Self { + Self { + session: self.session, + block_number: self.block_number, + } + } +} + +impl<'a, S: MorphProofsBatchSession> MorphProofsBatchTrieCursorFactory<'a, S> { + /// Initializes a session-scoped trie cursor factory. + pub const fn new(session: &'a S, block_number: u64) -> Self { + Self { + session, + block_number, + } + } +} + +impl TrieCursorFactory for MorphProofsBatchTrieCursorFactory<'_, S> +where + S: MorphProofsBatchSession, +{ + type AccountTrieCursor<'a> + = RawTrieCursor> + where + Self: 'a; + type StorageTrieCursor<'a> + = RawTrieCursor> + where + Self: 'a; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { + Ok(RawTrieCursor::new( + self.session + .account_trie_cursor(self.block_number) + .map_err(Into::::into)?, + )) + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(RawTrieCursor::new( + self.session + .storage_trie_cursor(hashed_address, self.block_number) + .map_err(Into::::into)?, + )) + } +} + +/// Session-scoped hashed cursor factory backed by a [`MorphProofsBatchSession`]. +#[derive(Debug)] +pub struct MorphProofsBatchHashedAccountCursorFactory<'a, S: MorphProofsBatchSession> { + session: &'a S, + block_number: u64, +} + +impl Clone for MorphProofsBatchHashedAccountCursorFactory<'_, S> { + fn clone(&self) -> Self { + Self { + session: self.session, + block_number: self.block_number, + } + } +} + +impl<'a, S: MorphProofsBatchSession> MorphProofsBatchHashedAccountCursorFactory<'a, S> { + /// Initializes a session-scoped hashed cursor factory. + pub const fn new(session: &'a S, block_number: u64) -> Self { + Self { + session, + block_number, + } + } +} + +impl HashedCursorFactory for MorphProofsBatchHashedAccountCursorFactory<'_, S> +where + S: MorphProofsBatchSession, +{ + type AccountCursor<'a> + = RawHashedAccountCursor> + where + Self: 'a; + type StorageCursor<'a> + = RawHashedStorageCursor> + where + Self: 'a; + + fn hashed_account_cursor(&self) -> Result, DatabaseError> { + Ok(RawHashedAccountCursor::new( + self.session.account_hashed_cursor(self.block_number)?, + )) + } + + fn hashed_storage_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(RawHashedStorageCursor::new( + self.session + .storage_hashed_cursor(hashed_address, self.block_number)?, + )) + } +} diff --git a/crates/proofs/src/db/batch.rs b/crates/proofs/src/db/batch.rs new file mode 100644 index 0000000..1b8ba86 --- /dev/null +++ b/crates/proofs/src/db/batch.rs @@ -0,0 +1,143 @@ +//! Batch write session for [`MdbxProofsStorage`] enabling multiple block writes inside one MDBX +//! RW transaction. Reads through the session observe uncommitted writes from earlier blocks in +//! the same session, which is required for cold catch-up where block `N+1` must execute against +//! block `N` written but not yet committed. + +use alloy_eips::eip1898::BlockWithParent; +use alloy_primitives::B256; +use reth_db::{ + Database, DatabaseEnv, + table::{DupSort, Table}, + transaction::DbTx, +}; + +use crate::{ + BlockStateDiff, MorphProofsStorageError, MorphProofsStorageResult, + api::{MorphProofsBatchSession, WriteCounts}, + db::{ + AccountTrieHistory, HashedAccountHistory, HashedStorageHistory, MdbxAccountCursor, + MdbxProofsStorage, MdbxStorageCursor, MdbxTrieCursor, StorageTrieHistory, + }, +}; + +/// Alias for the dup-sorted cursor type produced by an MDBX RW transaction. +pub type DupRw<'tx, T> = <::TXMut as DbTx>::DupCursor; + +/// Active write batch holding one MDBX RW transaction across multiple block writes. +#[derive(Debug)] +pub struct MdbxBatchSession<'tx> { + storage: &'tx MdbxProofsStorage, + tx: Option<::TXMut>, +} + +impl<'tx> MdbxBatchSession<'tx> { + pub(crate) const fn new( + storage: &'tx MdbxProofsStorage, + tx: ::TXMut, + ) -> Self { + Self { + storage, + tx: Some(tx), + } + } + + pub(crate) fn commit(mut self) -> MorphProofsStorageResult<()> { + if let Some(tx) = self.tx.take() { + tx.commit()?; + } + Ok(()) + } + + fn tx_ref(&self) -> MorphProofsStorageResult<&::TXMut> { + self.tx + .as_ref() + .ok_or(MorphProofsStorageError::BatchSessionClosed) + } + + fn dup_cursor(&self) -> MorphProofsStorageResult> { + Ok(self.tx_ref()?.cursor_dup_read::()?) + } +} + +impl MorphProofsBatchSession for MdbxBatchSession<'_> { + type StorageTrieCursor<'a> + = MdbxTrieCursor> + where + Self: 'a; + type AccountTrieCursor<'a> + = MdbxTrieCursor> + where + Self: 'a; + type StorageCursor<'a> + = MdbxStorageCursor> + where + Self: 'a; + type AccountHashedCursor<'a> + = MdbxAccountCursor> + where + Self: 'a; + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + self.storage + .inner_get_earliest_block_number_hash(self.tx_ref()?) + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + self.storage + .inner_get_latest_block_number_hash(self.tx_ref()?) + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxTrieCursor::new( + self.dup_cursor::()?, + max_block_number, + Some(hashed_address), + )) + } + + fn account_trie_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxTrieCursor::new( + self.dup_cursor::()?, + max_block_number, + None, + )) + } + + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxStorageCursor::new( + self.dup_cursor::()?, + max_block_number, + hashed_address, + )) + } + + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxAccountCursor::new( + self.dup_cursor::()?, + max_block_number, + )) + } + + fn store_trie_updates( + &mut self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + self.storage + .store_trie_updates_append_only(self.tx_ref()?, block_ref, block_state_diff) + } +} diff --git a/crates/proofs/src/db/cursor.rs b/crates/proofs/src/db/cursor.rs new file mode 100644 index 0000000..057a1a2 --- /dev/null +++ b/crates/proofs/src/db/cursor.rs @@ -0,0 +1,1545 @@ +use std::marker::PhantomData; + +use alloy_primitives::{B256, U256}; +use reth_db::{ + Database, DatabaseEnv, DatabaseError, + cursor::{DbCursorRO, DbDupCursorRO}, + table::{DupSort, Table}, + transaction::DbTx, +}; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{BranchNodeCompact, Nibbles, StoredNibbles}; + +use crate::{ + MorphProofsStorageResult, + db::{ + AccountTrieHistory, HashedAccountHistory, HashedStorageHistory, HashedStorageKey, + MaybeDeleted, StorageTrieHistory, StorageTrieKey, VersionedValue, + }, +}; + +/// Generic alias for dup cursor for T +pub type Dup<'tx, T> = <::TX as DbTx>::DupCursor; + +/// Iterates versioned dup-sorted rows and returns the latest value (<= `max_block_number`), +/// skipping tombstones. +#[derive(Debug, Clone)] +pub struct BlockNumberVersionedCursor { + _table: PhantomData, + cursor: Cursor, + max_block_number: u64, +} + +impl BlockNumberVersionedCursor +where + T: Table> + DupSort, + Cursor: DbCursorRO + DbDupCursorRO, +{ + /// Initializes new [`BlockNumberVersionedCursor`]. + pub const fn new(cursor: Cursor, max_block_number: u64) -> Self { + Self { + _table: PhantomData, + cursor, + max_block_number, + } + } + + /// Check if the cursor is currently positioned at a valid row. + fn is_positioned(&mut self) -> MorphProofsStorageResult { + Ok(self.cursor.current()?.is_some()) + } + + /// Resolve the latest version for `key` with `block_number` <= `max_block_number`. + /// Strategy: + /// - `seek_by_key_subkey(key, max)` gives first dup >= max. + /// - if exactly == max → it's our latest + /// - if > max → `prev_dup()` is latest < max (or None) + /// - if no dup >= max: + /// - if key exists → `last_dup()` is latest < max + /// - else → None + fn latest_version_for_key( + &mut self, + key: T::Key, + ) -> MorphProofsStorageResult> { + // First dup with subkey >= max_block_number + let seek_res = self + .cursor + .seek_by_key_subkey(key.clone(), self.max_block_number)?; + + if let Some(vv) = seek_res { + if vv.block_number > self.max_block_number { + // step back to the last dup < max + return Ok(self.cursor.prev_dup()?); + } + // already at the dup = max + return Ok(Some((key, vv))); + } + + // No dup >= max ⇒ either key absent or all dups < max. Check if key exists: + if self.cursor.seek_exact(key.clone())?.is_none() { + return Ok(None); + } + + // Key exists ⇒ take last dup (< max). + if let Some(vv) = self.cursor.last_dup()? { + return Ok(Some((key, vv))); + } + Ok(None) + } + + /// Returns a non-deleted latest version for exactly `key`, if any. + fn seek_exact(&mut self, key: T::Key) -> MorphProofsStorageResult> { + if let Some((latest_key, latest_value)) = self.latest_version_for_key(key)? + && let MaybeDeleted(Some(v)) = latest_value.value + { + return Ok(Some((latest_key, v))); + } + Ok(None) + } + + /// Walk forward from `first_key` (inclusive) until we find a *live* latest-≤-max value. + /// `first_key` must already be a *real key* in the table. + fn next_live_from( + &mut self, + mut first_key: T::Key, + ) -> MorphProofsStorageResult> { + loop { + // Compute latest version ≤ max for this key + if let Some((k, v)) = self.seek_exact(first_key.clone())? { + return Ok(Some((k, v))); + } + + // Move to next distinct key, or EOF + let Some((next_key, _)) = self.cursor.next_no_dup()? else { + return Ok(None); + }; + + first_key = next_key; + } + } + + /// Seek to the first non-deleted latest version at or after `start_key`. + /// Logic: + /// - Try exact key first (above). If alive, return it. + /// - Otherwise hop to next distinct key and repeat until we find a live version or hit EOF. + fn seek(&mut self, start_key: T::Key) -> MorphProofsStorageResult> { + // Position MDBX at first key >= start_key + if let Some((first_key, _)) = self.cursor.seek(start_key)? { + return self.next_live_from(first_key); + } + Ok(None) + } + + /// Advance to the next distinct key from the current MDBX position + /// and return its non-deleted latest version, if any. + /// Next distinct key; if not positioned, start from `T::Key::default()`. + fn next(&mut self) -> MorphProofsStorageResult> + where + T::Key: Default, + { + // If not positioned, start from the beginning (default key). + if self.cursor.current()?.is_none() { + let Some((first_key, _)) = self.cursor.seek(T::Key::default())? else { + return Ok(None); + }; + return self.next_live_from(first_key); + } + + // Otherwise advance to next distinct key and resume the walk. + let Some((next_key, _)) = self.cursor.next_no_dup()? else { + return Ok(None); + }; + self.next_live_from(next_key) + } +} + +/// MDBX implementation of [`TrieCursor`]. +#[derive(Debug)] +pub struct MdbxTrieCursor { + inner: BlockNumberVersionedCursor, + hashed_address: Option, +} + +impl< + V, + T: Table> + DupSort, + Cursor: DbCursorRO + DbDupCursorRO, +> MdbxTrieCursor +{ + /// Initializes new [`MdbxTrieCursor`]. + pub const fn new(cursor: Cursor, max_block_number: u64, hashed_address: Option) -> Self { + Self { + inner: BlockNumberVersionedCursor::new(cursor, max_block_number), + hashed_address, + } + } +} + +impl TrieCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn seek_exact( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + Ok(self + .inner + .seek_exact(StoredNibbles(path)) + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node)))?) + } + + fn seek( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + Ok(self + .inner + .seek(StoredNibbles(path)) + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node)))?) + } + + fn next(&mut self) -> Result, DatabaseError> { + Ok(self + .inner + .next() + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node)))?) + } + + fn current(&mut self) -> Result, DatabaseError> { + self.inner + .cursor + .current() + .map(|opt| opt.map(|(StoredNibbles(n), _)| n)) + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +impl TrieCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn seek_exact( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + let key = StorageTrieKey::new(address, StoredNibbles(path)); + return Ok(self.inner.seek_exact(key).map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + })?); + } + Ok(None) + } + + fn seek( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + let key = StorageTrieKey::new(address, StoredNibbles(path)); + return Ok(self.inner.seek(key).map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + })?); + } + Ok(None) + } + + fn next(&mut self) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + // If the cursor is not positioned, we need to seek to the first key for our bound + // address to ensure we start iterating from the correct position in the + // table. This is necessary because BlockNumberVersionedCursor::next() would + // otherwise start from T::Key::default() (the beginning of the entire + // table), which would cause us to miss entries for non-first addresses. + if !self.inner.is_positioned()? { + return self.seek(Nibbles::default()); + } + + return Ok(self.inner.next().map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + })?); + } + Ok(None) + } + + fn current(&mut self) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + return self.inner.cursor.current().map(|opt| { + opt.and_then(|(k, _)| (k.hashed_address == address).then_some(k.path.0)) + }); + } + Ok(None) + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +impl TrieStorageCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = Some(hashed_address); + } +} + +/// MDBX implementation of [`HashedCursor`] for storage state. +#[derive(Debug)] +pub struct MdbxStorageCursor { + inner: BlockNumberVersionedCursor, + hashed_address: B256, +} + +impl MdbxStorageCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + /// Initializes new [`MdbxStorageCursor`] + pub const fn new(cursor: Cursor, block_number: u64, hashed_address: B256) -> Self { + Self { + inner: BlockNumberVersionedCursor::new(cursor, block_number), + hashed_address, + } + } +} + +impl HashedCursor for MdbxStorageCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + type Value = U256; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + let storage_key = HashedStorageKey::new(self.hashed_address, key); + + // hashed storage values can be zero, which means the storage slot is deleted, so we should + // skip those + let result = self.inner.seek(storage_key).map(|opt| { + opt.and_then(|(k, v)| { + // Only return entries that belong to the bound address + (k.hashed_address == self.hashed_address).then_some((k.hashed_storage_key, v.0)) + }) + })?; + + if let Some((_, v)) = result + && v.is_zero() + { + return self.next(); + } + + Ok(result) + } + + fn next(&mut self) -> Result, DatabaseError> { + // If the cursor is not positioned, we need to seek to the first key for our bound address + // to ensure we start iterating from the correct position in the table. + // This is necessary because BlockNumberVersionedCursor::next() would otherwise start + // from T::Key::default() (the beginning of the entire table), which would cause us + // to miss entries for non-first addresses. + if !self.inner.is_positioned()? { + return self.seek(B256::ZERO); + } + + loop { + let result = self.inner.next().map(|opt| { + opt.and_then(|(k, v)| { + // Only return entries that belong to the bound address + (k.hashed_address == self.hashed_address).then_some((k.hashed_storage_key, v.0)) + }) + })?; + + // hashed storage values can be zero, which means the storage slot is deleted, so we + // should skip those + if let Some((_, v)) = result + && v.is_zero() + { + continue; + } + + return Ok(result); + } + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +impl HashedStorageCursor for MdbxStorageCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn is_storage_empty(&mut self) -> Result { + Ok(self.seek(B256::ZERO)?.is_none()) + } + + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = hashed_address + } +} + +/// MDBX implementation of [`HashedCursor`] for account state. +#[derive(Debug)] +pub struct MdbxAccountCursor { + inner: BlockNumberVersionedCursor, +} + +impl MdbxAccountCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + /// Initializes new `MdbxAccountCursor` + pub const fn new(cursor: Cursor, block_number: u64) -> Self { + Self { + inner: BlockNumberVersionedCursor::new(cursor, block_number), + } + } +} + +impl HashedCursor for MdbxAccountCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + type Value = Account; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + Ok(self.inner.seek(key)?) + } + + fn next(&mut self) -> Result, DatabaseError> { + Ok(self.inner.next()?) + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +#[cfg(test)] +mod tests { + use reth_db::{ + DatabaseEnv, + mdbx::{DatabaseArguments, init_db_for}, + }; + use reth_db_api::{ + Database, + cursor::DbDupCursorRW, + transaction::{DbTx, DbTxMut}, + }; + use reth_trie::{BranchNodeCompact, Nibbles, StoredNibbles}; + use tempfile::TempDir; + + use super::*; + use crate::db::{StorageValue, models}; + + fn setup_db() -> DatabaseEnv { + let tmp = TempDir::new().expect("create tmpdir"); + init_db_for::<_, models::Tables>(tmp, DatabaseArguments::default()).expect("init db") + } + + fn stored(path: Nibbles) -> StoredNibbles { + StoredNibbles(path) + } + + fn node() -> BranchNodeCompact { + BranchNodeCompact::default() + } + + fn append_account_trie( + wtx: &::TXMut, + key: StoredNibbles, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write cursor"); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val), + }; + c.append_dup(key, vv).expect("append dup"); + } + + fn append_storage_trie( + wtx: &::TXMut, + address: B256, + path: Nibbles, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write cursor"); + let key = StorageTrieKey::new(address, StoredNibbles(path)); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val), + }; + c.append_dup(key, vv).expect("append dup"); + } + + fn append_hashed_storage( + wtx: &::TXMut, + addr: B256, + slot: B256, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write"); + let key = HashedStorageKey::new(addr, slot); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val.map(StorageValue)), + }; + c.append_dup(key, vv).expect("append dup"); + } + + fn append_hashed_account( + wtx: &::TXMut, + key: B256, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write"); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val), + }; + c.append_dup(key, vv).expect("append dup"); + } + + // Open a dup-RO cursor and wrap it in a BlockNumberVersionedCursor with a given bound. + fn version_cursor( + tx: &::TX, + max_block: u64, + ) -> BlockNumberVersionedCursor> { + let cur = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + BlockNumberVersionedCursor::new(cur, max_block) + } + + fn account_trie_cursor( + tx: &'_ ::TX, + max_block: u64, + ) -> MdbxTrieCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + // For account trie the address is not used; pass None. + MdbxTrieCursor::new(c, max_block, None) + } + + // Helper: build a Storage trie cursor bound to an address + fn storage_trie_cursor( + tx: &'_ ::TX, + max_block: u64, + address: B256, + ) -> MdbxTrieCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + MdbxTrieCursor::new(c, max_block, Some(address)) + } + + fn storage_cursor( + tx: &'_ ::TX, + max_block: u64, + address: B256, + ) -> MdbxStorageCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + MdbxStorageCursor::new(c, max_block, address) + } + + fn account_cursor( + tx: &'_ ::TX, + max_block: u64, + ) -> MdbxAccountCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + MdbxAccountCursor::new(c, max_block) + } + + // Assert helper: ensure the chosen VersionedValue has the expected block and deletion flag. + fn assert_block( + got: Option<(StoredNibbles, VersionedValue)>, + expected_block: u64, + expect_deleted: bool, + ) { + let (_, vv) = got.expect("expected Some(..)"); + assert_eq!(vv.block_number, expected_block, "wrong block chosen"); + let is_deleted = matches!(vv.value, MaybeDeleted(None)); + assert_eq!(is_deleted, expect_deleted, "tombstone mismatch"); + } + + /// No entry for key → None. + #[test] + fn latest_version_for_key_none_when_key_absent() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cursor = version_cursor(&tx, 100); + + let out = cursor + .latest_version_for_key(stored(Nibbles::default())) + .expect("should not return error"); + assert!(out.is_none(), "absent key must return None"); + } + + /// Exact match at max (live) → pick it. + #[test] + fn latest_version_for_key_picks_value_at_max_if_present() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 50, Some(node())); // == max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 50, false); + } + + /// When `seek_by_key_subkey` points to the subkey > max - fallback to the prev. + #[test] + fn latest_version_for_key_picks_latest_below_max_when_next_is_above() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 30, Some(node())); // expected + append_account_trie(&wtx, k.clone(), 70, Some(node())); // > max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 30, false); + } + + /// No ≥ max but key exists → use last < max. + #[test] + fn latest_version_for_key_picks_last_below_max_when_none_at_or_above() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 40, Some(node())); // expected (max=100) + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 40, false); + } + + /// All entries are > max → None. + #[test] + fn latest_version_for_key_none_when_everything_is_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 60, Some(node())); + append_account_trie(&wtx, k1.clone(), 70, Some(node())); + append_account_trie(&wtx, k2, 40, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k1).expect("ok"); + assert!(out.is_none(), "no dup ≤ max ⇒ None"); + } + + /// Single dup < max → pick it. + #[test] + fn latest_version_for_key_picks_single_below_max() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 25, Some(node())); // < max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 25, false); + } + + /// Single dup == max → pick it. + #[test] + fn latest_version_for_key_picks_single_at_max() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 50, Some(node())); // == max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 50, false); + } + + /// Latest ≤ max is a tombstone → return it (this API doesn't filter). + #[test] + fn latest_version_for_key_returns_tombstone_if_latest_is_deleted() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 90, None); // latest ≤ max, but deleted + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 90, true); + } + + /// Should skip tombstones and return None when the latest ≤ max is deleted. + #[test] + fn seek_exact_skips_tombstone_returns_none() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 90, None); // latest ≤ max is tombstoned + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.seek_exact(k).expect("ok"); + assert!( + out.is_none(), + "seek_exact must filter out deleted latest value" + ); + } + + /// Empty table → None. + #[test] + fn seek_empty_returns_none() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.seek(stored(Nibbles::from_nibbles([0x0A]))).expect("ok"); + assert!(out.is_none()); + } + + /// Start at an existing key whose latest ≤ max is live → returns that key. + #[test] + fn seek_at_live_key_returns_it() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 20, Some(node())); // latest ≤ max + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k.clone()).expect("ok").expect("some"); + assert_eq!(out.0, k); + } + + /// Start at an existing key whose latest ≤ max is tombstoned → skip to next key with live + /// value. + #[test] + fn seek_skips_tombstoned_key_to_next_live_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + // Key 0x10 latest ≤ max is deleted + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + append_account_trie(&wtx, k1.clone(), 20, None); // tombstone at latest ≤ max + // Next key has live + append_account_trie(&wtx, k2.clone(), 5, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start between keys → returns the next key’s live latest ≤ max. + #[test] + fn seek_between_keys_returns_next_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0C])); + let k3 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2.clone(), 10, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Start at 0x15 (between 0x10 and 0x20) + + let out = cur.seek(k3).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start after the last key → None. + #[test] + fn seek_after_last_returns_none() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + let k3 = stored(Nibbles::from_nibbles([0x0C])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2, 10, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.seek(k3).expect("ok"); + assert!(out.is_none()); + } + + /// If the first key at-or-after has only versions > max, it is effectively not visible → skip + /// to next. + #[test] + fn seek_skips_keys_with_only_versions_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 60, Some(node())); + append_account_trie(&wtx, k2.clone(), 40, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start at a key with mixed versions; latest ≤ max is tombstone → skip to next key with live. + #[test] + fn seek_mixed_versions_tombstone_latest_skips_to_next_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + append_account_trie(&wtx, k1.clone(), 30, None); + append_account_trie(&wtx, k2.clone(), 5, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 30); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// When not positioned should start from default key and return the first live key. + #[test] + fn next_unpositioned_starts_from_default_returns_first_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); // first live + append_account_trie(&wtx, k2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + // Unpositioned cursor + let mut cur = version_cursor(&tx, 100); + + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k1); + } + + /// After positioning on a live key via `seek()`, `next()` should advance to the next live key. + #[test] + fn next_advances_from_current_live_to_next_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); // live + append_account_trie(&wtx, k2.clone(), 10, Some(node())); // next live + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Position at k1 + let _ = cur.seek(k1).expect("ok").expect("some"); + // Next should yield k2 + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// If the next key's latest ≤ max is tombstone, `next()` should skip to the next live key. + #[test] + fn next_skips_tombstoned_key_to_next_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); // will be tombstoned at latest ≤ max + let k3 = stored(Nibbles::from_nibbles([0x0C])); // next live + + { + let wtx = db.tx_mut().expect("rw tx"); + // k1 live + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + // k2: latest ≤ max is tombstone + append_account_trie(&wtx, k2.clone(), 10, Some(node())); + append_account_trie(&wtx, k2, 20, None); + // k3 live + append_account_trie(&wtx, k3.clone(), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + // Position at k1 + let _ = cur.seek(k1).expect("ok").expect("some"); + // next should skip k2 (tombstoned latest) and return k3 + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k3); + } + + /// If positioned on the last live key, `next()` should return None (EOF). + #[test] + fn next_returns_none_at_eof() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); // last key + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2.clone(), 10, Some(node())); // last live + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Position at the last key k2 + let _ = cur.seek(k2).expect("ok").expect("some"); + // `next()` should hit EOF + let out = cur.next().expect("ok"); + assert!(out.is_none()); + } + + /// If the first key has only versions > max, `next()` should skip it and return the next live + /// key. + #[test] + fn next_skips_keys_with_only_versions_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); // only > max + let k2 = stored(Nibbles::from_nibbles([0x0B])); // ≤ max live + + { + let wtx = db.tx_mut().expect("rw tx"); + // k1 only above max (max=50) + append_account_trie(&wtx, k1, 60, Some(node())); + // k2 within max + append_account_trie(&wtx, k2.clone(), 40, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + // Unpositioned; `next()` will start from default and walk + let mut cur = version_cursor(&tx, 50); + + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Empty table: `next()` should return None. + #[test] + fn next_on_empty_returns_none() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.next().expect("ok"); + assert!(out.is_none()); + } + + // ----------------- Account trie cursor thin-wrapper checks ----------------- + + #[test] + fn account_seek_exact_live_maps_key_and_value() { + let db = setup_db(); + let k = Nibbles::from_nibbles([0x0A]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + + // Build wrapper + let mut cur = account_trie_cursor(&tx, 100); + + // Wrapper should return (Nibbles, BranchNodeCompact) + let out = TrieCursor::seek_exact(&mut cur, k) + .expect("ok") + .expect("some"); + assert_eq!(out.0, k); + } + + #[test] + fn account_seek_exact_filters_tombstone() { + let db = setup_db(); + let k = Nibbles::from_nibbles([0x0B]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k), 5, Some(node())); + append_account_trie(&wtx, StoredNibbles(k), 9, None); // latest ≤ max tombstone + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = account_trie_cursor(&tx, 10); + + let out = TrieCursor::seek_exact(&mut cur, k).expect("ok"); + assert!(out.is_none(), "account seek_exact must filter tombstone"); + } + + #[test] + fn account_seek_and_next_and_current_roundtrip() { + let db = setup_db(); + let k1 = Nibbles::from_nibbles([0x01]); + let k2 = Nibbles::from_nibbles([0x02]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k1), 10, Some(node())); + append_account_trie(&wtx, StoredNibbles(k2), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = account_trie_cursor(&tx, 100); + + // seek at k1 + let out1 = TrieCursor::seek(&mut cur, k1).expect("ok").expect("some"); + assert_eq!(out1.0, k1); + + // current should be k1 + let cur_k = TrieCursor::current(&mut cur).expect("ok").expect("some"); + assert_eq!(cur_k, k1); + + // next should move to k2 + let out2 = TrieCursor::next(&mut cur).expect("ok").expect("some"); + assert_eq!(out2.0, k2); + } + + // ----------------- Storage trie cursor thin-wrapper checks ----------------- + + #[test] + fn storage_seek_exact_respects_address_filter() { + let db = setup_db(); + + let addr_a = B256::from([0xAA; 32]); + let addr_b = B256::from([0xBB; 32]); + + let path = Nibbles::from_nibbles([0x0D]); + + { + let wtx = db.tx_mut().expect("rw tx"); + // insert only under B + append_storage_trie(&wtx, addr_b, path, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + + // Cursor bound to A must not see B’s data + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + let out_a = TrieCursor::seek_exact(&mut cur_a, path).expect("ok"); + assert!(out_a.is_none(), "no data for addr A"); + + // Cursor bound to B should see it + let mut cur_b = storage_trie_cursor(&tx, 100, addr_b); + let out_b = TrieCursor::seek_exact(&mut cur_b, path) + .expect("ok") + .expect("some"); + assert_eq!(out_b.0, path); + } + + #[test] + fn storage_seek_returns_first_key_for_bound_address() { + let db = setup_db(); + + let addr_a = B256::from([0x11; 32]); + let addr_b = B256::from([0x22; 32]); + + let p1 = Nibbles::from_nibbles([0x01]); + let p2 = Nibbles::from_nibbles([0x02]); + let p3 = Nibbles::from_nibbles([0x03]); + + { + let wtx = db.tx_mut().expect("rw tx"); + // For A: only p2 + append_storage_trie(&wtx, addr_a, p2, 10, Some(node())); + // For B: p1 + append_storage_trie(&wtx, addr_b, p1, 10, Some(node())); + wtx.commit().expect("commit"); + } + + // test seek behaviour + { + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // seek at p1: for A there is no p1; the next key >= p1 under A is p2 + let out = TrieCursor::seek(&mut cur_a, p1).expect("ok").expect("some"); + assert_eq!(out.0, p2); + + // seek at p2: exact match + let out = TrieCursor::seek(&mut cur_a, p2).expect("ok").expect("some"); + assert_eq!(out.0, p2); + + // seek at p3: no p3 under A; no next key ≥ p3 under A → None + let out = TrieCursor::seek(&mut cur_a, p3).expect("ok"); + assert!(out.is_none(), "no key ≥ p3 under A"); + } + + // test next behaviour + { + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + let out = TrieCursor::next(&mut cur_a).expect("ok").expect("some"); + assert_eq!(out.0, p2); + + // next should yield None as there is no further key under A + let out = TrieCursor::next(&mut cur_a).expect("ok"); + assert!(out.is_none(), "no more keys under A"); + + // current should return None + let out = TrieCursor::current(&mut cur_a).expect("ok"); + assert!(out.is_none(), "no current key after EOF"); + } + + // test seek_exact behaviour + { + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // seek_exact at p1: no exact match + let out = TrieCursor::seek_exact(&mut cur_a, p1).expect("ok"); + assert!(out.is_none(), "no exact p1 under A"); + + // seek_exact at p2: exact match + let out = TrieCursor::seek_exact(&mut cur_a, p2) + .expect("ok") + .expect("some"); + assert_eq!(out.0, p2); + + // seek_exact at p3: no exact match + let out = TrieCursor::seek_exact(&mut cur_a, p3).expect("ok"); + assert!(out.is_none(), "no exact p3 under A"); + } + } + + #[test] + fn storage_next_stops_at_address_boundary() { + let db = setup_db(); + + let addr_a = B256::from([0x33; 32]); + let addr_b = B256::from([0x44; 32]); + + let p1 = Nibbles::from_nibbles([0x05]); // under A + let p2 = Nibbles::from_nibbles([0x06]); // under B (next key overall) + + { + let wtx = db.tx_mut().expect("rw tx"); + append_storage_trie(&wtx, addr_a, p1, 10, Some(node())); + append_storage_trie(&wtx, addr_b, p2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // position at p1 (A) + let _ = TrieCursor::seek_exact(&mut cur_a, p1) + .expect("ok") + .expect("some"); + + // next should reach boundary; impl filters different address and returns None + let out = TrieCursor::next(&mut cur_a).expect("ok"); + assert!( + out.is_none(), + "next() should stop when next key is a different address" + ); + } + + #[test] + fn storage_current_maps_key() { + let db = setup_db(); + + let addr = B256::from([0x55; 32]); + let p = Nibbles::from_nibbles([0x09]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_storage_trie(&wtx, addr, p, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = storage_trie_cursor(&tx, 100, addr); + + let _ = TrieCursor::seek_exact(&mut cur, p) + .expect("ok") + .expect("some"); + + let now = TrieCursor::current(&mut cur).expect("ok").expect("some"); + assert_eq!(now, p); + } + + #[test] + fn hashed_storage_seek_maps_slot_and_value() { + let db = setup_db(); + let addr = B256::from([0xAA; 32]); + let slot = B256::from([0x10; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr, slot, 10, Some(U256::from(7))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 100, addr); + + let (got_slot, got_val) = cur.seek(slot).expect("ok").expect("some"); + assert_eq!(got_slot, slot); + assert_eq!(got_val, U256::from(7)); + } + + #[test] + fn hashed_storage_seek_filters_tombstone() { + let db = setup_db(); + let addr = B256::from([0xAB; 32]); + let slot = B256::from([0x11; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr, slot, 5, Some(U256::from(1))); + append_hashed_storage(&wtx, addr, slot, 9, None); // latest ≤ max is tombstone + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 10, addr); + + let out = cur.seek(slot).expect("ok"); + assert!(out.is_none(), "wrapper must filter tombstoned latest"); + } + + #[test] + fn hashed_storage_seek_and_next_roundtrip() { + let db = setup_db(); + let addr = B256::from([0xAC; 32]); + let s1 = B256::from([0x01; 32]); + let s2 = B256::from([0x02; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr, s1, 10, Some(U256::from(11))); + append_hashed_storage(&wtx, addr, s2, 10, Some(U256::from(22))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 100, addr); + + let (k1, v1) = cur.seek(s1).expect("ok").expect("some"); + assert_eq!((k1, v1), (s1, U256::from(11))); + + let (k2, v2) = cur.next().expect("ok").expect("some"); + assert_eq!((k2, v2), (s2, U256::from(22))); + } + + #[test] + fn hashed_storage_address_boundary() { + let db = setup_db(); + let addr1 = B256::from([0xAC; 32]); + let addr2 = B256::from([0xAD; 32]); + let s1 = B256::from([0x01; 32]); + let s2 = B256::from([0x02; 32]); + let s3 = B256::from([0x03; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr1, s1, 10, Some(U256::from(11))); + append_hashed_storage(&wtx, addr1, s2, 10, Some(U256::from(22))); + wtx.commit().expect("commit"); + } + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr2, s1, 10, Some(U256::from(33))); + append_hashed_storage(&wtx, addr2, s2, 10, Some(U256::from(44))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 100, addr1); + + let (k1, v1) = cur.next().expect("ok").expect("some"); + assert_eq!((k1, v1), (s1, U256::from(11))); + + let (k2, v2) = cur.next().expect("ok").expect("some"); + assert_eq!((k2, v2), (s2, U256::from(22))); + + let out = cur.next().expect("ok"); + assert!(out.is_none(), "should stop at address boundary"); + + let (k1, v1) = cur.seek(s1).expect("ok").expect("some"); + assert_eq!((k1, v1), (s1, U256::from(11))); + + let (k2, v2) = cur.seek(s2).expect("ok").expect("some"); + assert_eq!((k2, v2), (s2, U256::from(22))); + + let out = cur.seek(s3).expect("ok"); + assert!(out.is_none(), "should not see keys from other address"); + } + + #[test] + fn hashed_account_seek_maps_key_and_value() { + let db = setup_db(); + let key = B256::from([0x20; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_account(&wtx, key, 10, Some(Account::default())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = account_cursor(&tx, 100); + + let (got_key, _acc) = cur.seek(key).expect("ok").expect("some"); + assert_eq!(got_key, key); + } + + #[test] + fn hashed_account_seek_filters_tombstone() { + let db = setup_db(); + let key = B256::from([0x21; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_account(&wtx, key, 5, Some(Account::default())); + append_hashed_account(&wtx, key, 9, None); // latest ≤ max is tombstone + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = account_cursor(&tx, 10); + + let out = cur.seek(key).expect("ok"); + assert!(out.is_none(), "wrapper must filter tombstoned latest"); + } + + #[test] + fn hashed_account_seek_and_next_roundtrip() { + let db = setup_db(); + let k1 = B256::from([0x01; 32]); + let k2 = B256::from([0x02; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_account(&wtx, k1, 10, Some(Account::default())); + append_hashed_account(&wtx, k2, 10, Some(Account::default())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = account_cursor(&tx, 100); + + let (got1, _) = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(got1, k1); + + let (got2, _) = cur.next().expect("ok").expect("some"); + assert_eq!(got2, k2); + } + + /// Regression test: `MdbxStorageCursor` `next()` should work without explicit `seek()` + /// when cursor is constructed for a non-first key. + /// + /// Bug: When a storage cursor is created for a specific address (e.g., 0x02), + /// calling `next()` without first calling `seek()` returns None instead of the first + /// slot for that address. This only manifests when the address is not the first + /// in the table. + #[test] + fn storage_cursor_next_without_seek_for_non_first_address() { + let db = setup_db(); + let addr1 = B256::from([0x01; 32]); // First address + let addr2 = B256::from([0x02; 32]); // Second address (non-first) + let slot1 = B256::from([0x11; 32]); + let slot2 = B256::from([0x12; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + // Add storage for first address + append_hashed_storage(&wtx, addr1, slot1, 10, Some(U256::from(100))); + + // Add storage for second address + append_hashed_storage(&wtx, addr2, slot2, 10, Some(U256::from(200))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + + // Test with addr1 (first address) - this typically works + let mut cur1 = storage_cursor(&tx, 100, addr1); + let result1 = cur1.next().expect("ok"); + assert!( + result1.is_some(), + "next() should return data for first address without seek()" + ); + if let Some((key, val)) = result1 { + assert_eq!(key, slot1); + assert_eq!(val, U256::from(100)); + } + + // Test with addr2 (non-first address) - this demonstrates the bug fix + let mut cur2 = storage_cursor(&tx, 100, addr2); + let result2_without_seek = cur2.next().expect("ok"); + + assert!( + result2_without_seek.is_some(), + "next() should return data for non-first address without seek()" + ); + if let Some((key, val)) = result2_without_seek { + assert_eq!(key, slot2); + assert_eq!(val, U256::from(200)); + } + + // Verify that seek() works correctly + let mut cur3 = storage_cursor(&tx, 100, addr2); + let result3_with_seek = cur3.seek(slot2).expect("ok"); + assert!( + result3_with_seek.is_some(), + "seek() should find the slot for addr2" + ); + if let Some((key, val)) = result3_with_seek { + assert_eq!(key, slot2); + assert_eq!(val, U256::from(200)); + } + } + + /// Regression test: `MdbxTrieCursor` `next()` should work without `seek()` + /// for non-first addresses. + #[test] + fn storage_trie_cursor_next_without_seek_for_non_first_address() { + let db = setup_db(); + let addr1 = B256::from([0x01; 32]); + let addr2 = B256::from([0x02; 32]); + let path1 = Nibbles::from_nibbles([0x0A]); + let path2 = Nibbles::from_nibbles([0x0B]); + + { + let wtx = db.tx_mut().expect("rw"); + append_storage_trie(&wtx, addr1, path1, 10, Some(node())); + append_storage_trie(&wtx, addr2, path2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + + // Test addr1 (first) - works + let mut cur1 = storage_trie_cursor(&tx, 100, addr1); + let result1 = TrieCursor::next(&mut cur1).expect("ok"); + assert!(result1.is_some()); + assert_eq!(result1.unwrap().0, path1); + + // Test addr2 (non-first) - should also work now + let mut cur2 = storage_trie_cursor(&tx, 100, addr2); + let result2 = TrieCursor::next(&mut cur2).expect("ok"); + assert!( + result2.is_some(), + "next() should work for non-first address without seek()" + ); + assert_eq!(result2.unwrap().0, path2); + } +} diff --git a/crates/proofs/src/db/mod.rs b/crates/proofs/src/db/mod.rs new file mode 100644 index 0000000..7ebcfc3 --- /dev/null +++ b/crates/proofs/src/db/mod.rs @@ -0,0 +1,15 @@ +//! MDBX-backed implementation of [`MorphProofsStore`](crate::MorphProofsStore). + +mod models; +pub use models::*; + +mod store; +pub use store::{MdbxProofsStorage, MdbxProofsStorageOptions, ProofDbIdentity, ProofWindowBounds}; + +mod cursor; +pub use cursor::{ + BlockNumberVersionedCursor, Dup, MdbxAccountCursor, MdbxStorageCursor, MdbxTrieCursor, +}; + +mod batch; +pub use batch::{DupRw, MdbxBatchSession}; diff --git a/crates/proofs/src/db/models/block.rs b/crates/proofs/src/db/models/block.rs new file mode 100644 index 0000000..7475c36 --- /dev/null +++ b/crates/proofs/src/db/models/block.rs @@ -0,0 +1,82 @@ +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use bytes::BufMut; +use derive_more::{From, Into}; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decompress}, +}; +use serde::{Deserialize, Serialize}; + +/// Wrapper for block number and block hash tuple to implement [`Compress`]/[`Decompress`]. +/// +/// Used for storing block metadata (number + hash). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, From, Into)] +pub struct BlockNumberHash(BlockNumHash); + +impl Compress for BlockNumberHash { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + // Encode block number (8 bytes, big-endian) + hash (32 bytes) = 40 bytes total + buf.put_u64(self.0.number); + buf.put_slice(self.0.hash.as_slice()); + } +} + +impl Decompress for BlockNumberHash { + fn decompress(value: &[u8]) -> Result { + if value.len() != 40 { + return Err(DecompressError::new(DatabaseError::Decode)); + } + + let number = u64::from_be_bytes( + value[..8] + .try_into() + .map_err(|_| DecompressError::new(DatabaseError::Decode))?, + ); + let hash = B256::from_slice(&value[8..40]); + + Ok(Self(BlockNumHash { number, hash })) + } +} + +impl BlockNumberHash { + /// Create new instance. + pub const fn new(number: u64, hash: B256) -> Self { + Self(BlockNumHash { number, hash }) + } + + /// Get the block number. + pub const fn number(&self) -> u64 { + self.0.number + } + + /// Get the block hash. + pub const fn hash(&self) -> &B256 { + &self.0.hash + } +} + +#[cfg(test)] +mod tests { + use alloy_primitives::B256; + + use super::*; + + #[test] + fn test_block_number_hash_roundtrip() { + let test_cases = vec![ + BlockNumberHash::new(0, B256::ZERO), + BlockNumberHash::new(42, B256::repeat_byte(0xaa)), + BlockNumberHash::new(u64::MAX, B256::repeat_byte(0xff)), + ]; + + for original in test_cases { + let compressed = original.compress(); + let decompressed = BlockNumberHash::decompress(&compressed).unwrap(); + assert_eq!(original, decompressed); + } + } +} diff --git a/crates/proofs/src/db/models/change_set.rs b/crates/proofs/src/db/models/change_set.rs new file mode 100644 index 0000000..d9f7d80 --- /dev/null +++ b/crates/proofs/src/db/models/change_set.rs @@ -0,0 +1,131 @@ +use alloy_primitives::B256; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{self, Decode, Encode}, +}; +use reth_trie_common::StoredNibbles; +use serde::{Deserialize, Serialize}; + +use crate::db::{HashedStorageKey, StorageTrieKey}; + +/// The keys of the entries in the history tables. +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct ChangeSet { + /// Keys changed in [`AccountTrieHistory`](super::AccountTrieHistory) table. + pub account_trie_keys: Vec, + /// Keys changed in [`StorageTrieHistory`](super::StorageTrieHistory) table. + pub storage_trie_keys: Vec, + /// Keys changed in [`HashedAccountHistory`](super::HashedAccountHistory) table. + pub hashed_account_keys: Vec, + /// Keys changed in [`HashedStorageHistory`](super::HashedStorageHistory) table. + pub hashed_storage_keys: Vec, +} + +impl table::Encode for ChangeSet { + type Encoded = Vec; + + fn encode(self) -> Self::Encoded { + bincode::serde::encode_to_vec(&self, bincode::config::standard()) + .expect("ChangeSet serialization should not fail") + } +} + +impl table::Decode for ChangeSet { + fn decode(value: &[u8]) -> Result { + bincode::serde::decode_from_slice(value, bincode::config::standard()) + .map(|(v, _)| v) + .map_err(|_| DatabaseError::Decode) + } +} + +impl table::Compress for ChangeSet { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + let encoded = self.clone().encode(); + buf.put_slice(&encoded); + } +} + +impl table::Decompress for ChangeSet { + fn decompress(value: &[u8]) -> Result { + Self::decode(value).map_err(DecompressError::new) + } +} + +#[cfg(test)] +mod tests { + use alloy_primitives::B256; + use reth_db::table::{Compress, Decompress}; + + use super::*; + + #[test] + fn test_encode_decode_empty_change_set() { + let change_set = ChangeSet { + account_trie_keys: vec![], + storage_trie_keys: vec![], + hashed_account_keys: vec![], + hashed_storage_keys: vec![], + }; + + let encoded = change_set.clone().encode(); + let decoded = ChangeSet::decode(&encoded).expect("Failed to decode"); + assert_eq!(change_set, decoded); + } + + #[test] + fn test_encode_decode_populated_change_set() { + let account_key = StoredNibbles::from(vec![1, 2, 3, 4]); + let storage_key = StorageTrieKey { + hashed_address: B256::repeat_byte(0x11), + path: StoredNibbles::from(vec![5, 6, 7, 8]), + }; + let hashed_storage_key = HashedStorageKey { + hashed_address: B256::repeat_byte(0x22), + hashed_storage_key: B256::repeat_byte(0x33), + }; + + let change_set = ChangeSet { + account_trie_keys: vec![account_key], + storage_trie_keys: vec![storage_key], + hashed_account_keys: vec![B256::repeat_byte(0x44)], + hashed_storage_keys: vec![hashed_storage_key], + }; + + let encoded = change_set.clone().encode(); + let decoded = ChangeSet::decode(&encoded).expect("Failed to decode"); + assert_eq!(change_set, decoded); + } + + #[test] + fn test_decode_invalid_data() { + let invalid_data = vec![0xFF; 32]; + let result = ChangeSet::decode(&invalid_data); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), DatabaseError::Decode)); + } + + #[test] + fn test_compress_decompress() { + let change_set = ChangeSet { + account_trie_keys: vec![StoredNibbles::from(vec![1, 2, 3])], + storage_trie_keys: vec![StorageTrieKey { + hashed_address: B256::ZERO, + path: StoredNibbles::from(vec![4, 5, 6]), + }], + hashed_account_keys: vec![B256::ZERO], + hashed_storage_keys: vec![HashedStorageKey { + hashed_address: B256::ZERO, + hashed_storage_key: B256::repeat_byte(0x42), + }], + }; + + let mut buf = Vec::new(); + change_set.compress_to_buf(&mut buf); + + let decompressed = ChangeSet::decompress(&buf).expect("Failed to decompress"); + assert_eq!(change_set, decompressed); + } +} diff --git a/crates/proofs/src/db/models/kv.rs b/crates/proofs/src/db/models/kv.rs new file mode 100644 index 0000000..05f8580 --- /dev/null +++ b/crates/proofs/src/db/models/kv.rs @@ -0,0 +1,85 @@ +use alloy_primitives::B256; +use reth_db::table::{DupSort, Table}; +use reth_primitives_traits::Account; +use reth_trie_common::{BranchNodeCompact, Nibbles, StoredNibbles}; + +use crate::db::{ + AccountTrieHistory, HashedAccountHistory, HashedStorageHistory, HashedStorageKey, MaybeDeleted, + StorageTrieHistory, StorageTrieKey, StorageValue, VersionedValue, +}; + +/// Helper to convert inputs into a table key or kv pair. +pub trait IntoKV { + /// Convert `self` into the table key. + fn into_key(self) -> Tab::Key; + + /// Convert `self` into kv for the given `block_number`. + fn into_kv(self, block_number: u64) -> (Tab::Key, Tab::Value); +} + +impl IntoKV for (Nibbles, Option) { + fn into_key(self) -> StoredNibbles { + StoredNibbles::from(self.0) + } + + fn into_kv(self, block_number: u64) -> (StoredNibbles, VersionedValue) { + let (path, node) = self; + ( + StoredNibbles::from(path), + VersionedValue { + block_number, + value: MaybeDeleted(node), + }, + ) + } +} + +impl IntoKV for (B256, Nibbles, Option) { + fn into_key(self) -> StorageTrieKey { + let (hashed_address, path, _) = self; + StorageTrieKey::new(hashed_address, StoredNibbles::from(path)) + } + fn into_kv(self, block_number: u64) -> (StorageTrieKey, VersionedValue) { + let (hashed_address, path, node) = self; + ( + StorageTrieKey::new(hashed_address, StoredNibbles::from(path)), + VersionedValue { + block_number, + value: MaybeDeleted(node), + }, + ) + } +} + +impl IntoKV for (B256, Option) { + fn into_key(self) -> B256 { + self.0 + } + fn into_kv(self, block_number: u64) -> (B256, VersionedValue) { + let (hashed_address, account) = self; + ( + hashed_address, + VersionedValue { + block_number, + value: MaybeDeleted(account), + }, + ) + } +} + +impl IntoKV for (B256, B256, Option) { + fn into_key(self) -> HashedStorageKey { + let (hashed_address, hashed_storage_key, _) = self; + HashedStorageKey::new(hashed_address, hashed_storage_key) + } + fn into_kv(self, block_number: u64) -> (HashedStorageKey, VersionedValue) { + let (hashed_address, hashed_storage_key, value) = self; + ( + HashedStorageKey::new(hashed_address, hashed_storage_key), + VersionedValue { + block_number, + value: MaybeDeleted(value), + }, + ) + } +} diff --git a/crates/proofs/src/db/models/metadata.rs b/crates/proofs/src/db/models/metadata.rs new file mode 100644 index 0000000..999795e --- /dev/null +++ b/crates/proofs/src/db/models/metadata.rs @@ -0,0 +1,56 @@ +use bytes::BufMut; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decode, Decompress, Encode}, +}; +use serde::{Deserialize, Serialize}; + +/// Durable proof-database metadata keys. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum ProofMetadataKey { + /// EIP-155 chain identifier. + ChainId = 0, + /// Canonical genesis block hash. + GenesisHash = 1, + /// Proof-database schema version. + SchemaVersion = 2, +} + +impl Encode for ProofMetadataKey { + type Encoded = [u8; 1]; + + fn encode(self) -> Self::Encoded { + [self as u8] + } +} + +impl Decode for ProofMetadataKey { + fn decode(value: &[u8]) -> Result { + match value.first() { + Some(0) => Ok(Self::ChainId), + Some(1) => Ok(Self::GenesisHash), + Some(2) => Ok(Self::SchemaVersion), + _ => Err(DatabaseError::Decode), + } + } +} + +/// Opaque metadata bytes whose shape is validated by the store. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProofMetadataValue(pub Vec); + +impl Compress for ProofMetadataValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + buf.put_slice(&self.0); + } +} + +impl Decompress for ProofMetadataValue { + fn decompress(value: &[u8]) -> Result { + Ok(Self(value.to_vec())) + } +} diff --git a/crates/proofs/src/db/models/mod.rs b/crates/proofs/src/db/models/mod.rs new file mode 100644 index 0000000..de5b978 --- /dev/null +++ b/crates/proofs/src/db/models/mod.rs @@ -0,0 +1,93 @@ +//! MDBX implementation of [`MorphProofsStore`](crate::MorphProofsStore). +//! +//! This module provides a complete MDBX implementation of the +//! [`MorphProofsStore`](crate::MorphProofsStore) trait. It uses the [`reth_db`] crate for +//! database interactions and defines the necessary tables and models for storing trie branches, +//! accounts, and storage leaves. + +mod block; +pub use block::*; +mod version; +pub use version::*; +mod storage; +pub use storage::*; +mod change_set; +mod kv; +mod metadata; +use std::fmt; + +use alloy_primitives::B256; +pub use change_set::*; +pub use kv::*; +pub use metadata::*; +use reth_db::{ + TableSet, TableType, TableViewer, + table::{DupSort, TableInfo}, + tables, +}; +use reth_primitives_traits::Account; +use reth_trie_common::{BranchNodeCompact, StoredNibbles}; + +tables! { + /// Stores historical branch nodes for the account state trie. + /// + /// Each entry maps a compact-encoded trie path (`StoredNibbles`) to its versioned branch node. + /// Multiple versions of the same node are stored using the block number as a subkey. + table AccountTrieHistory { + type Key = StoredNibbles; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores historical branch nodes for the storage trie of each account. + /// + /// Each entry is identified by a composite key combining the account’s hashed address and the + /// compact-encoded trie path. Versions are tracked using block numbers as subkeys. + table StorageTrieHistory { + type Key = StorageTrieKey; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores versioned account state across block history. + /// + /// Each entry maps a hashed account address to its serialized account data (balance, nonce, + /// code hash, storage root). + table HashedAccountHistory { + type Key = B256; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores versioned storage state across block history. + /// + /// Each entry maps a composite key of (hashed address, storage key) to its stored value. + /// Used for reconstructing contract storage at any historical block height. + table HashedStorageHistory { + type Key = HashedStorageKey; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Tracks the active proof window in the external historical storage. + /// + /// Stores the earliest and latest block numbers (and corresponding hashes) + /// for which historical trie data is retained. + table ProofWindow { + type Key = ProofWindowKey; + type Value = BlockNumberHash; + } + + /// Stores schema and canonical-chain identity for fail-fast cold-copy validation. + table ProofMetadata { + type Key = ProofMetadataKey; + type Value = ProofMetadataValue; + } + + /// A reverse mapping of block numbers to a keys of the tables. + /// This is used for efficiently locating data by block number. + table BlockChangeSet { + type Key = u64; // Block number + type Value = ChangeSet; + } +} diff --git a/crates/proofs/src/db/models/storage.rs b/crates/proofs/src/db/models/storage.rs new file mode 100644 index 0000000..bcf30df --- /dev/null +++ b/crates/proofs/src/db/models/storage.rs @@ -0,0 +1,272 @@ +use alloy_primitives::{B256, U256}; +use derive_more::{Constructor, From, Into}; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decode, Decompress, Encode}, +}; +use reth_trie_common::StoredNibbles; +use serde::{Deserialize, Serialize}; + +/// Composite key: `(hashed-address, path)` for storage trie branches +/// +/// Used to efficiently index storage branches by both account address and trie path. +/// The encoding ensures lexicographic ordering: first by address, then by path. +#[derive(Default, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct StorageTrieKey { + /// Hashed account address + pub hashed_address: B256, + /// Trie path as nibbles + pub path: StoredNibbles, +} + +impl StorageTrieKey { + /// Create a new storage branch key + pub const fn new(hashed_address: B256, path: StoredNibbles) -> Self { + Self { + hashed_address, + path, + } + } +} + +impl Encode for StorageTrieKey { + type Encoded = Vec; + + fn encode(self) -> Self::Encoded { + let mut buf = Vec::with_capacity(32 + self.path.0.len()); + // First encode the address (32 bytes) + buf.extend_from_slice(self.hashed_address.as_slice()); + // Then encode the path + buf.extend_from_slice(&self.path.encode()); + buf + } +} + +impl Decode for StorageTrieKey { + fn decode(value: &[u8]) -> Result { + if value.len() < 32 { + return Err(DatabaseError::Decode); + } + + // First 32 bytes are the address + let hashed_address = B256::from_slice(&value[..32]); + + // Remaining bytes are the path + let path = StoredNibbles::decode(&value[32..])?; + + Ok(Self { + hashed_address, + path, + }) + } +} + +/// Composite key: (`hashed_address`, `hashed_storage_key`) for hashed storage values +/// +/// Used to efficiently index storage values by both account address and storage key. +/// The encoding ensures lexicographic ordering: first by address, then by storage key. +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct HashedStorageKey { + /// Hashed account address + pub hashed_address: B256, + /// Hashed storage key + pub hashed_storage_key: B256, +} + +impl HashedStorageKey { + /// Create a new hashed storage key + pub const fn new(hashed_address: B256, hashed_storage_key: B256) -> Self { + Self { + hashed_address, + hashed_storage_key, + } + } +} + +impl Encode for HashedStorageKey { + type Encoded = [u8; 64]; + + fn encode(self) -> Self::Encoded { + let mut buf = [0u8; 64]; + // First 32 bytes: address + buf[..32].copy_from_slice(self.hashed_address.as_slice()); + // Next 32 bytes: storage key + buf[32..].copy_from_slice(self.hashed_storage_key.as_slice()); + buf + } +} + +impl Decode for HashedStorageKey { + fn decode(value: &[u8]) -> Result { + if value.len() != 64 { + return Err(DatabaseError::Decode); + } + + let hashed_address = B256::from_slice(&value[..32]); + let hashed_storage_key = B256::from_slice(&value[32..64]); + + Ok(Self { + hashed_address, + hashed_storage_key, + }) + } +} + +/// Storage value wrapper for U256 values +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, From, Into, Constructor)] +pub struct StorageValue(pub U256); + +impl Compress for StorageValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + let be: [u8; 32] = self.0.to_be_bytes::<32>(); + buf.put_slice(&be); + } +} + +impl Decompress for StorageValue { + fn decompress(value: &[u8]) -> Result { + if value.len() != 32 { + return Err(DecompressError::new(DatabaseError::Decode)); + } + let bytes: [u8; 32] = value + .try_into() + .map_err(|_| DecompressError::new(DatabaseError::Decode))?; + Ok(Self(U256::from_be_bytes(bytes))) + } +} + +/// Proof Window key for tracking active proof window bounds +/// +/// Used to store earliest and latest block numbers in the external storage. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum ProofWindowKey { + /// Earliest block number stored in external storage + EarliestBlock = 0, + /// Latest block number stored in external storage + LatestBlock = 1, + /// Anchor block from where the initial state initialization started + InitialStateAnchor = 2, +} + +impl Encode for ProofWindowKey { + type Encoded = [u8; 1]; + + fn encode(self) -> Self::Encoded { + [self as u8] + } +} + +impl Decode for ProofWindowKey { + fn decode(value: &[u8]) -> Result { + match value.first() { + Some(&0) => Ok(Self::EarliestBlock), + Some(&1) => Ok(Self::LatestBlock), + Some(&2) => Ok(Self::InitialStateAnchor), + _ => Err(DatabaseError::Decode), + } + } +} + +#[cfg(test)] +mod tests { + use reth_trie::Nibbles; + + use super::*; + + #[test] + fn test_storage_branch_subkey_encode_decode() { + let addr = B256::from([1u8; 32]); + let path = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 2, 3, 4])); + let key = StorageTrieKey::new(addr, path.clone()); + + let encoded = key.clone().encode(); + let decoded = StorageTrieKey::decode(&encoded).unwrap(); + + assert_eq!(key, decoded); + assert_eq!(decoded.hashed_address, addr); + assert_eq!(decoded.path, path); + } + + #[test] + fn test_storage_branch_subkey_ordering() { + let addr1 = B256::from([1u8; 32]); + let addr2 = B256::from([2u8; 32]); + let path1 = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 2])); + let path2 = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 3])); + + let key1 = StorageTrieKey::new(addr1, path1.clone()); + let key2 = StorageTrieKey::new(addr1, path2); + let key3 = StorageTrieKey::new(addr2, path1); + + // Encoded bytes should be sortable: first by address, then by path + let enc1 = key1.encode(); + let enc2 = key2.encode(); + let enc3 = key3.encode(); + + assert!(enc1 < enc2, "Same address, path1 < path2"); + assert!(enc1 < enc3, "addr1 < addr2"); + assert!(enc2 < enc3, "addr1 < addr2 (even with larger path)"); + } + + #[test] + fn test_hashed_storage_subkey_encode_decode() { + let addr = B256::from([1u8; 32]); + let storage_key = B256::from([2u8; 32]); + let key = HashedStorageKey::new(addr, storage_key); + + let encoded = key.clone().encode(); + let decoded = HashedStorageKey::decode(&encoded).unwrap(); + + assert_eq!(key, decoded); + assert_eq!(decoded.hashed_address, addr); + assert_eq!(decoded.hashed_storage_key, storage_key); + } + + #[test] + fn test_hashed_storage_subkey_ordering() { + let addr1 = B256::from([1u8; 32]); + let addr2 = B256::from([2u8; 32]); + let storage1 = B256::from([10u8; 32]); + let storage2 = B256::from([20u8; 32]); + + let key1 = HashedStorageKey::new(addr1, storage1); + let key2 = HashedStorageKey::new(addr1, storage2); + let key3 = HashedStorageKey::new(addr2, storage1); + + // Encoded bytes should be sortable: first by address, then by storage key + let enc1 = key1.encode(); + let enc2 = key2.encode(); + let enc3 = key3.encode(); + + assert!(enc1 < enc2, "Same address, storage1 < storage2"); + assert!(enc1 < enc3, "addr1 < addr2"); + assert!(enc2 < enc3, "addr1 < addr2 (even with larger storage key)"); + } + + #[test] + fn test_hashed_storage_subkey_size() { + let addr = B256::from([1u8; 32]); + let storage_key = B256::from([2u8; 32]); + let key = HashedStorageKey::new(addr, storage_key); + + let encoded = key.encode(); + assert_eq!(encoded.len(), 64, "Encoded size should be exactly 64 bytes"); + } + + #[test] + fn test_metadata_key_encode_decode() { + let key = ProofWindowKey::EarliestBlock; + let encoded = key.encode(); + let decoded = ProofWindowKey::decode(&encoded).unwrap(); + assert_eq!(key, decoded); + + let key = ProofWindowKey::LatestBlock; + let encoded = key.encode(); + let decoded = ProofWindowKey::decode(&encoded).unwrap(); + assert_eq!(key, decoded); + } +} diff --git a/crates/proofs/src/db/models/version.rs b/crates/proofs/src/db/models/version.rs new file mode 100644 index 0000000..5e1ea7d --- /dev/null +++ b/crates/proofs/src/db/models/version.rs @@ -0,0 +1,205 @@ +use bytes::{Buf, BufMut}; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decompress}, +}; +use reth_primitives_traits::ValueWithSubKey; +use serde::{Deserialize, Serialize}; + +/// Wrapper type for `Option` that implements [`Compress`] and [`Decompress`] +/// +/// Encoding: +/// - `None` => empty byte array (length 0) +/// - `Some(value)` => compressed bytes of value (length > 0) +/// +/// This assumes the inner type `T` always compresses to non-empty bytes when it exists. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaybeDeleted(pub Option); + +impl From> for MaybeDeleted { + fn from(opt: Option) -> Self { + Self(opt) + } +} + +impl From> for Option { + fn from(maybe: MaybeDeleted) -> Self { + maybe.0 + } +} + +impl Compress for MaybeDeleted { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + match &self.0 { + None => { + // Empty = deleted, write nothing + } + Some(value) => { + // Compress the inner value to the buffer + value.compress_to_buf(buf); + } + } + } +} + +impl Decompress for MaybeDeleted { + fn decompress(value: &[u8]) -> Result { + if value.is_empty() { + // Empty = deleted + Ok(Self(None)) + } else { + // Non-empty = present + let inner = T::decompress(value)?; + Ok(Self(Some(inner))) + } + } +} + +/// Versioned value wrapper for [`DupSort`] tables +/// +/// For [`DupSort`] tables in MDBX, the Value type must contain the [`DupSort::SubKey`] as a field. +/// This wrapper combines a [`block_number`] (the [`DupSort::SubKey`]) with +/// the actual value. +/// +/// [`DupSort`]: reth_db::table::DupSort +/// [`DupSort::SubKey`]: reth_db::table::DupSort::SubKey +/// [`block_number`]: Self::block_number +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VersionedValue { + /// Block number ([`DupSort::SubKey`] for [`DupSort`]) + /// + /// [`DupSort`]: reth_db::table::DupSort + /// [`DupSort::SubKey`]: reth_db::table::DupSort::SubKey + pub block_number: u64, + /// The actual value (may be deleted) + pub value: MaybeDeleted, +} + +impl VersionedValue { + /// Create a new versioned value + pub const fn new(block_number: u64, value: MaybeDeleted) -> Self { + Self { + block_number, + value, + } + } +} + +impl Compress for VersionedValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + // Encode block number first (8 bytes, big-endian) + buf.put_u64(self.block_number); + // Then encode the value + self.value.compress_to_buf(buf); + } +} + +impl Decompress for VersionedValue { + fn decompress(value: &[u8]) -> Result { + if value.len() < 8 { + return Err(DecompressError::new(DatabaseError::Decode)); + } + + let mut buf: &[u8] = value; + let block_number = buf.get_u64(); + let value = MaybeDeleted::::decompress(&value[8..])?; + + Ok(Self { + block_number, + value, + }) + } +} + +impl ValueWithSubKey for VersionedValue { + type SubKey = u64; + + fn get_subkey(&self) -> Self::SubKey { + self.block_number + } +} + +#[cfg(test)] +mod tests { + use reth_primitives_traits::Account; + use reth_trie::BranchNodeCompact; + + use super::*; + + #[test] + fn test_maybe_deleted_none() { + let none: MaybeDeleted = MaybeDeleted(None); + let compressed = none.compress(); + assert!(compressed.is_empty(), "None should compress to empty bytes"); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, None); + } + + #[test] + fn test_maybe_deleted_some_account() { + let account = Account { + nonce: 42, + balance: alloy_primitives::U256::from(1000u64), + bytecode_hash: None, + }; + let some = MaybeDeleted(Some(account)); + let compressed = some.compress(); + assert!( + !compressed.is_empty(), + "Some should compress to non-empty bytes" + ); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, Some(account)); + } + + #[test] + fn test_maybe_deleted_some_branch() { + // Create a simple valid BranchNodeCompact (empty is valid) + let branch = BranchNodeCompact::new( + 0, // state_mask + 0, // tree_mask + 0, // hash_mask + vec![], // hashes + None, // root_hash + ); + let some = MaybeDeleted(Some(branch.clone())); + let compressed = some.compress(); + assert!( + !compressed.is_empty(), + "Some should compress to non-empty bytes" + ); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, Some(branch)); + } + + #[test] + fn test_maybe_deleted_roundtrip() { + let test_cases = vec![ + MaybeDeleted(None), + MaybeDeleted(Some(Account { + nonce: 0, + balance: alloy_primitives::U256::ZERO, + bytecode_hash: None, + })), + MaybeDeleted(Some(Account { + nonce: 999, + balance: alloy_primitives::U256::MAX, + bytecode_hash: Some([0xff; 32].into()), + })), + ]; + + for original in test_cases { + let compressed = original.clone().compress(); + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(original, decompressed); + } + } +} diff --git a/crates/proofs/src/db/store.rs b/crates/proofs/src/db/store.rs new file mode 100644 index 0000000..0f7064d --- /dev/null +++ b/crates/proofs/src/db/store.rs @@ -0,0 +1,4707 @@ +use std::{collections::BTreeMap, ops::RangeBounds, path::Path, time::Duration}; + +use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; +use alloy_primitives::{B256, U256, map::HashMap}; +use reth_db::{ + Database, DatabaseEnv, DatabaseError, + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, + mdbx::{DatabaseArguments, MaxReadTransactionDuration, init_db_for}, + table::{DupSort, Table}, + transaction::{DbTx, DbTxMut}, +}; +use reth_primitives_traits::Account; +use reth_trie::{hashed_cursor::HashedCursor, trie_cursor::TrieCursor}; +use reth_trie_common::{ + BranchNodeCompact, HashedPostState, Nibbles, StoredNibbles, + updates::{StorageTrieUpdates, TrieUpdates}, +}; + +use super::{ + BlockNumberHash, ProofMetadata, ProofMetadataKey, ProofMetadataValue, ProofWindow, + ProofWindowKey, Tables, +}; +use crate::{ + BlockStateDiff, MorphProofsStorageError, + MorphProofsStorageError::NoBlocksFound, + MorphProofsStorageResult, MorphProofsStore, + api::{ + InitialStateAnchor, InitialStateStatus, MorphProofsBatchStore, + MorphProofsInitialStateStore, WriteCounts, + }, + db::{ + MdbxAccountCursor, MdbxBatchSession, MdbxStorageCursor, MdbxTrieCursor, + cursor::Dup, + models::{ + AccountTrieHistory, BlockChangeSet, ChangeSet, HashedAccountHistory, + HashedStorageHistory, HashedStorageKey, IntoKV, MaybeDeleted, StorageTrieHistory, + StorageTrieKey, StorageValue, VersionedValue, + }, + }, +}; + +/// MDBX implementation of [`MorphProofsStore`]. +#[derive(Debug)] +pub struct MdbxProofsStorage { + env: DatabaseEnv, +} + +/// Options for opening [`MdbxProofsStorage`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MdbxProofsStorageOptions { + /// Maximum duration of a read transaction. + pub max_read_transaction_duration: Option, +} + +/// Canonical-chain identity persisted in the proof database. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProofDbIdentity { + /// EIP-155 chain identifier. + pub chain_id: u64, + /// Canonical genesis block hash. + pub genesis_hash: B256, +} + +impl ProofDbIdentity { + /// Creates a proof-database identity. + pub const fn new(chain_id: u64, genesis_hash: B256) -> Self { + Self { + chain_id, + genesis_hash, + } + } +} + +impl Default for ProofDbIdentity { + fn default() -> Self { + Self::new(0, B256::ZERO) + } +} + +/// Inclusive range currently served by the proof database. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProofWindowBounds { + /// Oldest block whose proof data is retained. + pub earliest: (u64, B256), + /// Newest block durably indexed by proof history. + pub latest: (u64, B256), +} + +/// Current on-disk schema. There is intentionally no migration path before launch. +const SCHEMA_VERSION: u32 = 1; + +fn decode_u64_metadata( + value: ProofMetadataValue, + field: &'static str, +) -> MorphProofsStorageResult { + let bytes: [u8; 8] = value + .0 + .try_into() + .map_err(|_| MorphProofsStorageError::CorruptMetadata(field))?; + Ok(u64::from_be_bytes(bytes)) +} + +fn decode_u32_metadata( + value: ProofMetadataValue, + field: &'static str, +) -> MorphProofsStorageResult { + let bytes: [u8; 4] = value + .0 + .try_into() + .map_err(|_| MorphProofsStorageError::CorruptMetadata(field))?; + Ok(u32::from_be_bytes(bytes)) +} + +fn decode_hash_metadata( + value: ProofMetadataValue, + field: &'static str, +) -> MorphProofsStorageResult { + let bytes: [u8; 32] = value + .0 + .try_into() + .map_err(|_| MorphProofsStorageError::CorruptMetadata(field))?; + Ok(B256::new(bytes)) +} + +struct ProofWindowValue { + earliest: NumHash, + latest: NumHash, +} + +/// Preprocessed prune plan for a target block number +#[derive(Debug, Clone)] +struct PrunePlan { + earliest_block: u64, + acc_survivors: Vec<(StoredNibbles, u64)>, + storage_survivors: Vec<(StorageTrieKey, u64)>, + hashed_acc_survivors: Vec<(B256, u64)>, + hashed_storage_survivors: Vec<(HashedStorageKey, u64)>, +} + +/// Preprocessed delete work for a prune range +#[derive(Debug, Default, Clone)] +struct HistoryDeleteBatch { + account_trie: Vec<(::Key, u64)>, + storage_trie: Vec<(::Key, u64)>, + hashed_account: Vec<(::Key, u64)>, + hashed_storage: Vec<(::Key, u64)>, +} + +impl MdbxProofsStorage { + /// Creates a new [`MdbxProofsStorage`] instance with the given path. + pub fn new(path: &Path) -> Result { + Self::open(path, ProofDbIdentity::default()) + } + + /// Opens an MDBX proof database and validates its canonical-chain identity. + pub fn open(path: &Path, identity: ProofDbIdentity) -> Result { + Self::open_with_options(path, identity, MdbxProofsStorageOptions::default()) + } + + /// Creates a new [`MdbxProofsStorage`] instance with the given path and options. + pub fn new_with_options( + path: &Path, + options: MdbxProofsStorageOptions, + ) -> Result { + Self::open_with_options(path, ProofDbIdentity::default(), options) + } + + /// Opens an MDBX proof database with runtime options and validates chain identity. + pub fn open_with_options( + path: &Path, + identity: ProofDbIdentity, + options: MdbxProofsStorageOptions, + ) -> Result { + let mut args = DatabaseArguments::default(); + if let Some(duration) = options.max_read_transaction_duration { + args = args.with_max_read_transaction_duration(Some(MaxReadTransactionDuration::Set( + duration, + ))); + } + let env = init_db_for::<_, Tables>(path, args) + .map_err(|e| DatabaseError::Other(format!("Failed to open database: {e}")))?; + let storage = Self { env }; + storage.validate_or_initialize_identity(identity)?; + Ok(storage) + } + + fn validate_or_initialize_identity( + &self, + identity: ProofDbIdentity, + ) -> MorphProofsStorageResult<()> { + // Use one explicit write transaction so the empty-database check and identity + // initialization are atomic. Dropping the transaction on any error aborts it instead of + // committing a partially-written identity. + let tx = self.env.tx_mut()?; + let stored_chain_id = tx.get::(ProofMetadataKey::ChainId)?; + let stored_genesis_hash = tx.get::(ProofMetadataKey::GenesisHash)?; + let stored_schema_version = tx.get::(ProofMetadataKey::SchemaVersion)?; + let metadata_entries = tx.entries::()?; + + match (stored_chain_id, stored_genesis_hash, stored_schema_version) { + (None, None, None) if metadata_entries == 0 => { + if !Self::proof_tables_are_empty(&tx)? { + return Err(MorphProofsStorageError::ProofDataWithoutIdentityMetadata); + } + tx.put::( + ProofMetadataKey::ChainId, + ProofMetadataValue(identity.chain_id.to_be_bytes().to_vec()), + )?; + tx.put::( + ProofMetadataKey::GenesisHash, + ProofMetadataValue(identity.genesis_hash.as_slice().to_vec()), + )?; + tx.put::( + ProofMetadataKey::SchemaVersion, + ProofMetadataValue(SCHEMA_VERSION.to_be_bytes().to_vec()), + )?; + } + (Some(chain_id), Some(genesis_hash), Some(schema_version)) if metadata_entries == 3 => { + let chain_id = decode_u64_metadata(chain_id, "chain_id")?; + if chain_id != identity.chain_id { + return Err(MorphProofsStorageError::ChainIdentityMismatch("chain_id")); + } + + let genesis_hash = decode_hash_metadata(genesis_hash, "genesis_hash")?; + if genesis_hash != identity.genesis_hash { + return Err(MorphProofsStorageError::ChainIdentityMismatch( + "genesis_hash", + )); + } + + let schema_version = decode_u32_metadata(schema_version, "schema_version")?; + if schema_version != SCHEMA_VERSION { + return Err(MorphProofsStorageError::SchemaMismatch { + expected: SCHEMA_VERSION, + actual: schema_version, + }); + } + } + _ => return Err(MorphProofsStorageError::IncompleteIdentityMetadata), + } + + tx.commit()?; + Ok(()) + } + + fn proof_tables_are_empty(tx: &impl DbTx) -> Result { + Ok(tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0) + } + + /// Returns the inclusive range currently served by proof history. + pub fn proof_window(&self) -> MorphProofsStorageResult> { + self.env.view(|tx| { + Ok(self + .inner_get_proof_window(tx)? + .map(|window| ProofWindowBounds { + earliest: (window.earliest.number, window.earliest.hash), + latest: (window.latest.number, window.latest.hash), + })) + })? + } + + pub(crate) fn inner_get_latest_block_number_hash( + &self, + tx: &impl DbTx, + ) -> MorphProofsStorageResult> { + let block = self.inner_get_block_number_hash(tx, ProofWindowKey::LatestBlock)?; + if block.is_some() { + return Ok(block); + } + + self.inner_get_block_number_hash(tx, ProofWindowKey::EarliestBlock) + } + + pub(crate) fn inner_get_earliest_block_number_hash( + &self, + tx: &impl DbTx, + ) -> MorphProofsStorageResult> { + self.inner_get_block_number_hash(tx, ProofWindowKey::EarliestBlock) + } + + fn inner_get_block_number_hash( + &self, + tx: &impl DbTx, + key: ProofWindowKey, + ) -> MorphProofsStorageResult> { + let mut cursor = tx.cursor_read::()?; + let value = cursor.seek_exact(key)?; + Ok(value.map(|(_, val)| (val.number(), *val.hash()))) + } + + fn inner_get_proof_window( + &self, + tx: &impl DbTx, + ) -> MorphProofsStorageResult> { + let mut cursor = tx.cursor_read::()?; + + let earliest = match cursor.seek_exact(ProofWindowKey::EarliestBlock)? { + Some((_, val)) => NumHash::new(val.number(), *val.hash()), + None => return Ok(None), + }; + + let latest = match cursor.seek_exact(ProofWindowKey::LatestBlock)? { + Some((_, val)) => NumHash::new(val.number(), *val.hash()), + None => earliest, + }; + + Ok(Some(ProofWindowValue { earliest, latest })) + } + + fn set_earliest_block_number_hash( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let _ = self.env.update(|tx| { + Self::inner_set_earliest_block_number(tx, block_number, hash)?; + Ok::<(), DatabaseError>(()) + })?; + Ok(()) + } + + /// Internal helper to set earliest block number hash within an existing transaction + fn inner_set_earliest_block_number( + tx: &(impl DbTxMut + DbTx), + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let mut cursor = tx.cursor_write::()?; + cursor.upsert( + ProofWindowKey::EarliestBlock, + &BlockNumberHash::new(block_number, hash), + )?; + Ok(()) + } + + /// Internal helper to set latest block number hash within an existing transaction + fn inner_set_latest_block_number( + tx: &(impl DbTxMut + DbTx), + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let mut cursor = tx.cursor_write::()?; + cursor.upsert( + ProofWindowKey::LatestBlock, + &BlockNumberHash::new(block_number, hash), + )?; + Ok(()) + } + + /// Persist a batch of versioned history entries to a dup-sorted table. + /// + /// # Parameters + /// - `block_number`: Target block number for versioning entries + /// - `items`: **Must be sorted** - iterator of entries to persist + /// - `append_mode`: Mode selector for write strategy: + /// - `true` (Append): Appends all entries including tombstones for forward progress + /// - `false` (Prune): Removes tombstones, writes non-tombstones to block 0 + /// + /// The cost of pruning is the cost of (append + deleting tombstones + deleting old block 0). + /// The tombstones deletion is expensive as it requires a seek for each (key + subkey). + /// + /// Uses [`reth_db::mdbx::cursor::Cursor::upsert`] for upsert operation. + fn persist_history_batch( + &self, + tx: &(impl DbTxMut + DbTx), + block_number: T::SubKey, + items: I, + append_mode: bool, + ) -> MorphProofsStorageResult> + where + T: Table> + DupSort, + T::Key: Clone, + I: IntoIterator, + I::Item: IntoKV, + { + let mut cur = tx.cursor_dup_write::()?; + let mut keys = Vec::::new(); + + // Materialize iterator to enable partitioning and collect keys + let mut pairs: Vec<(T::Key, T::Value)> = Vec::new(); + for it in items { + let (k, vv) = it.into_kv(block_number); + pairs.push((k.clone(), vv)); + keys.push(k) + } + + if append_mode { + // Append all entries (including tombstones) to preserve full history + for (k, vv) in pairs { + cur.append_dup(k.clone(), vv)?; + } + return Ok(keys); + } + + // Drop current cursor to start clean for Phase 1 + drop(cur); + + // Phase 1: Batch Delete (Sequential) + // Remove all existing state at Block 0 for these keys. + { + let mut del_cur = tx.cursor_dup_write::()?; + for (k, _) in &pairs { + // Seek to (Key, Block 0) + if let Some(vv) = del_cur.seek_by_key_subkey(k.clone(), 0)? + && vv.block_number == 0 + { + del_cur.delete_current()?; + } + } + } + + // Phase 2: Batch Write (Sequential) + // Write new values (skipping tombstones). + { + let mut write_cur = tx.cursor_dup_write::()?; + for (k, vv) in pairs { + if vv.value.0.is_some() { + write_cur.upsert(k, &vv)?; + } + } + } + + Ok(keys) + } + + /// Delete entries for `items` at exactly `block_number` in a dup-sorted table. + /// Seeks (key, block) and deletes current if the subkey matches. + fn delete_dup_sorted( + &self, + tx: &(impl DbTxMut + DbTx), + items: I, + ) -> MorphProofsStorageResult<()> + where + T: Table> + DupSort, + T::Key: Clone, + T::SubKey: PartialEq + Clone, + I: IntoIterator, + { + let mut cur = tx.cursor_dup_write::()?; + for (key, subkey) in items { + if let Some(vv) = cur.seek_by_key_subkey(key, subkey)? { + // ensure we didn't land on a >subkey + if vv.block_number == subkey { + cur.delete_current()?; + } + } + } + Ok(()) + } + + /// Phase 1 of pruning: Calculate survivors. + /// Scans change sets to find the LATEST update for every key in the range. + fn calculate_prune_plan( + &self, + target_block: u64, + ) -> MorphProofsStorageResult> { + self.env.view(|tx| { + let Some((earliest, _)) = + self.inner_get_block_number_hash(tx, ProofWindowKey::EarliestBlock)? + else { + return Ok(None); + }; + + if earliest >= target_block { + return Ok(None); + } + + // Walk the change set in REVERSE so the first time we see a key, its block_number + // is by definition the latest version in [earliest+1, target_block] — the only + // one that must survive. This lets us use `or_insert` instead of an + // `and_modify(max).or_insert` per key, halving the per-insert work. + // + // HashMaps are pre-sized to bound rehash overhead. The chosen capacity is a rough + // average of unique keys per block on Base mainnet; the maps grow if exceeded. + const PER_BLOCK_CAPACITY_HINT: usize = 4096; + let blocks_in_range = target_block + .saturating_sub(earliest) + .try_into() + .unwrap_or(usize::MAX); + let cap = blocks_in_range + .saturating_mul(PER_BLOCK_CAPACITY_HINT) + .min(1 << 20); + + let mut acc_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + let mut storage_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + let mut hashed_acc_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + let mut hashed_storage_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + + let mut cs_cursor = tx.cursor_read::()?; + let mut walker = cs_cursor.walk_back(Some(target_block))?; + while let Some(Ok((block_number, cs))) = walker.next() { + if block_number <= earliest { + break; + } + for k in cs.account_trie_keys { + acc_candidates.entry(k).or_insert(block_number); + } + for k in cs.storage_trie_keys { + storage_candidates.entry(k).or_insert(block_number); + } + for k in cs.hashed_account_keys { + hashed_acc_candidates.entry(k).or_insert(block_number); + } + for k in cs.hashed_storage_keys { + hashed_storage_candidates.entry(k).or_insert(block_number); + } + } + + Ok(Some(PrunePlan { + earliest_block: earliest, + acc_survivors: Self::flatten_and_sort(acc_candidates), + storage_survivors: Self::flatten_and_sort(storage_candidates), + hashed_acc_survivors: Self::flatten_and_sort(hashed_acc_candidates), + hashed_storage_survivors: Self::flatten_and_sort(hashed_storage_candidates), + })) + })? + } + + /// Helper to flatten `HashMap` into a sorted Vector of survivors. + /// Sorting is required to ensure optimal sequential seek performance in MDBX. + fn flatten_and_sort(map: HashMap) -> Vec<(K, u64)> { + let mut v: Vec<_> = map.into_iter().collect(); + v.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + v + } + + /// Delete history versions for `items` that are strictly older than the provided block number. + /// `items` is a list of (Key, `SurvivorBlock`). Everything strictly older than `SurvivorBlock` + /// is deleted. Returns the number of entries deleted. + fn prune_history_preceding( + &self, + tx: &(impl DbTxMut + DbTx), + cutoff_items: Vec<(T::Key, u64)>, + ) -> MorphProofsStorageResult + where + T: Table> + DupSort, + T::Key: Clone + Ord, + { + if cutoff_items.is_empty() { + return Ok(0); + } + + let mut deleted_count = 0; + let mut cur = tx.cursor_dup_write::()?; + for (key, survivor_block) in cutoff_items { + let Some(mut entry) = cur.seek_by_key_subkey(key, 0)? else { + continue; + }; + loop { + if entry.block_number >= survivor_block { + // Reached the survivor version (or newer). Stop deleting for this key. + // If the survivor is a tombstone (None), delete it too — with all older + // history already gone, a tombstone at the new earliest is redundant + // (it implies "does not exist"). + if entry.block_number == survivor_block && entry.value.0.is_none() { + cur.delete_current()?; + deleted_count += 1; + } + break; + } + + // Entry is strictly older than survivor. Delete it. + cur.delete_current()?; + deleted_count += 1; + + // libmdbx semantics: after `delete_current()` the cursor sits directly + // before the just-deleted slot, so `next_dup_val` (MDBX_NEXT_DUP) yields the + // following dup of the same key, or `None` once this key's dup-group is + // exhausted. That bound removes the need for an explicit key comparison. + match cur.next_dup_val()? { + Some(next_entry) => entry = next_entry, + None => break, + } + } + } + Ok(deleted_count) + } + + /// Tombstone every key returned by `next`, then overlay `new_entries` so collisions + /// resolve in favor of `new_entries`, and `append_dup` the merged set in sorted-key order at + /// `block_number`. Used by the wipe branches of `store_trie_updates_for_block` to keep new + /// slots / nodes that arrive in the same block as a `wiped`/`is_deleted` entry — + /// `HashedStorage::from_plain_storage` sets `wiped = was_destroyed()` (true for + /// `DestroyedChanged`), so a SELFDESTRUCT + same-block recreate produces both at once. + fn wipe_and_overlay( + &self, + tx: &(impl DbTxMut + DbTx), + block_number: u64, + hashed_address: B256, + mut next: Next, + new_entries: I, + ) -> MorphProofsStorageResult> + where + T: Table> + DupSort, + Next: FnMut() -> MorphProofsStorageResult>, + I: IntoIterator)>, + (B256, K, Option): IntoKV, + T::Key: Clone, + K: Ord, + { + let mut merged: BTreeMap> = BTreeMap::new(); + while let Some((k, _vv)) = next()? { + merged.insert(k, None); + } + for (k, v) in new_entries { + merged.insert(k, v); + } + + let mut cur = tx.cursor_dup_write::()?; + let mut keys: Vec = Vec::with_capacity(merged.len()); + for (k, value) in merged { + let key: T::Key = (hashed_address, k, Option::::None).into_key(); + let vv: T::Value = VersionedValue { + block_number, + value: MaybeDeleted(value), + }; + cur.append_dup(key.clone(), vv)?; + keys.push(key); + } + Ok(keys) + } + + /// Collect versioned history over `block_range` using `BlockChangeSet`. + fn collect_history_ranged( + &self, + tx: &impl DbTx, + block_range: impl RangeBounds, + ) -> MorphProofsStorageResult { + let mut history = HistoryDeleteBatch::default(); + let mut change_set_cursor = tx.cursor_read::()?; + let mut walker = change_set_cursor.walk_range(block_range)?; + + while let Some(Ok((block_number, change_set))) = walker.next() { + // Push (key, subkey=block_number) pairs + history.account_trie.extend( + change_set + .account_trie_keys + .into_iter() + .map(|k| (k, block_number)), + ); + history.storage_trie.extend( + change_set + .storage_trie_keys + .into_iter() + .map(|k| (k, block_number)), + ); + history.hashed_account.extend( + change_set + .hashed_account_keys + .into_iter() + .map(|k| (k, block_number)), + ); + history.hashed_storage.extend( + change_set + .hashed_storage_keys + .into_iter() + .map(|k| (k, block_number)), + ); + } + + // Sorting by tuple sorts by key first, then by block_number. + history + .account_trie + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + history + .storage_trie + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + history + .hashed_account + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + history + .hashed_storage + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + + Ok(history) + } + + /// Delete versioned history over `block_range` using history batch. + fn delete_history_ranged( + &self, + tx: &(impl DbTxMut + DbTx), + block_range: impl RangeBounds, + history: HistoryDeleteBatch, + ) -> MorphProofsStorageResult { + let mut change_set_cursor = tx.cursor_write::()?; + let mut walker = change_set_cursor.walk_range(block_range)?; + + while let Some(Ok((_, _))) = walker.next() { + walker.delete_current()?; + } + + // Delete using the simplified API: iterator of (key, subkey) + self.delete_dup_sorted::(tx, history.clone().account_trie)?; + self.delete_dup_sorted::(tx, history.clone().storage_trie)?; + self.delete_dup_sorted::(tx, history.clone().hashed_account)?; + self.delete_dup_sorted::(tx, history.clone().hashed_storage)?; + + Ok(WriteCounts { + account_trie_updates_written_total: history.account_trie.len() as u64, + storage_trie_updates_written_total: history.storage_trie.len() as u64, + hashed_accounts_written_total: history.hashed_account.len() as u64, + hashed_storages_written_total: history.hashed_storage.len() as u64, + }) + } + + /// Write trie/state history for `block_number` from `block_state_diff`. + fn store_trie_updates_for_block( + &self, + tx: &::TXMut, + block_number: u64, + block_state_diff: BlockStateDiff, + append_mode: bool, + ) -> MorphProofsStorageResult { + let BlockStateDiff { + sorted_trie_updates, + sorted_post_state, + } = block_state_diff; + + let storage_trie_len = sorted_trie_updates.storage_tries_ref().len(); + let hashed_storage_len = sorted_post_state.storages.len(); + + let account_trie_keys = self.persist_history_batch( + tx, + block_number, + sorted_trie_updates.account_nodes_ref().iter().cloned(), + append_mode, + )?; + let hashed_account_keys = self.persist_history_batch( + tx, + block_number, + sorted_post_state.accounts.iter().copied(), + append_mode, + )?; + + let mut storage_trie_keys = Vec::::with_capacity(storage_trie_len); + for (hashed_address, nodes) in sorted_trie_updates.storage_tries_ref() { + if nodes.is_deleted && append_mode { + // Lookback must use the active tx so it sees uncommitted writes from earlier + // blocks in the same batch session. Opening a fresh RO tx here would read the + // pre-batch committed snapshot and emit wrong tombstones for wipe blocks whose + // parent is staged in the same batch. + let mut ro = MdbxTrieCursor::::new( + tx.cursor_dup_read::()?, + block_number - 1, + Some(*hashed_address), + ); + let keys = self.wipe_and_overlay( + tx, + block_number, + *hashed_address, + || Ok(ro.next()?), + nodes.storage_nodes_ref().iter().cloned(), + )?; + storage_trie_keys.extend(keys); + continue; + } + + let keys = self.persist_history_batch( + tx, + block_number, + nodes + .storage_nodes_ref() + .iter() + .cloned() + .map(|(path, node)| (*hashed_address, path, node)), + append_mode, + )?; + storage_trie_keys.extend(keys); + } + + let mut hashed_storage_keys = Vec::::with_capacity(hashed_storage_len); + for (hashed_address, storage) in sorted_post_state.storages { + if append_mode && storage.is_wiped() { + // See lookback note above: must use the active tx, not a fresh RO tx. + let mut ro = MdbxStorageCursor::new( + tx.cursor_dup_read::()?, + block_number - 1, + hashed_address, + ); + let keys = self.wipe_and_overlay( + tx, + block_number, + hashed_address, + || Ok(ro.next()?), + storage + .storage_slots_ref() + .iter() + .map(|(slot, val)| (*slot, Some(StorageValue(*val)))), + )?; + hashed_storage_keys.extend(keys); + continue; + } + let keys = self.persist_history_batch( + tx, + block_number, + storage + .storage_slots_ref() + .iter() + .map(|(key, val)| (hashed_address, *key, Some(StorageValue(*val)))), + append_mode, + )?; + hashed_storage_keys.extend(keys); + } + + Ok(ChangeSet { + account_trie_keys, + storage_trie_keys, + hashed_account_keys, + hashed_storage_keys, + }) + } + + /// Append-only writer for a block: validates parent, persists diff (soft-delete=true), + /// records a `BlockChangeSet`, and advances `ProofWindow::LatestBlock`. + pub(crate) fn store_trie_updates_append_only( + &self, + tx: &::TXMut, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + let block_number = block_ref.block.number; + + // Check the latest stored block is the parent of the incoming block + let latest_block_hash = self + .inner_get_latest_block_number_hash(tx)? + .map_or(B256::ZERO, |(_num, hash)| hash); + + if latest_block_hash != block_ref.parent { + return Err(MorphProofsStorageError::OutOfOrder { + block_number, + parent_block_hash: block_ref.parent, + latest_block_hash, + }); + } + + let change_set = + &self.store_trie_updates_for_block(tx, block_number, block_state_diff, true)?; + + // Cursor for recording all changes made in this block for all history tables + let mut change_set_cursor = tx.new_cursor::()?; + change_set_cursor.append(block_number, change_set)?; + + // Update proof window's latest block + Self::inner_set_latest_block_number(tx, block_number, block_ref.block.hash)?; + + Ok(WriteCounts { + account_trie_updates_written_total: change_set.account_trie_keys.len() as u64, + storage_trie_updates_written_total: change_set.storage_trie_keys.len() as u64, + hashed_accounts_written_total: change_set.hashed_account_keys.len() as u64, + hashed_storages_written_total: change_set.hashed_storage_keys.len() as u64, + }) + } + + /// Return `BlockNumHash` for the initial state anchor. + fn get_initial_state_anchor(&self) -> MorphProofsStorageResult> { + self.env.view(|tx| { + let mut cur = tx.cursor_read::()?; + Ok(cur + .seek_exact(ProofWindowKey::InitialStateAnchor)? + .map(|(_k, v)| v.into())) + })? + } + + /// Return latest key for a table + fn get_latest_key(&self) -> MorphProofsStorageResult> + where + T: Table, + { + self.env.view(|tx| { + let mut cursor = tx.cursor_read::()?; + Ok(cursor.last()?.map(|(k, _)| k)) + })? + } +} + +impl MorphProofsStore for MdbxProofsStorage { + type StorageTrieCursor<'tx> + = MdbxTrieCursor> + where + Self: 'tx; + type AccountTrieCursor<'tx> + = MdbxTrieCursor> + where + Self: 'tx; + type StorageCursor<'tx> + = MdbxStorageCursor> + where + Self: 'tx; + type AccountHashedCursor<'tx> + = MdbxAccountCursor> + where + Self: 'tx; + type Tx<'tx> + = ::TX + where + Self: 'tx; + + fn ro_tx<'tx>(&'tx self) -> MorphProofsStorageResult> { + Ok(self.env.tx()?) + } + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + self.env + .view(|tx| self.inner_get_earliest_block_number_hash(tx))? + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + self.env + .view(|tx| self.inner_get_latest_block_number_hash(tx))? + } + + fn get_earliest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.inner_get_earliest_block_number_hash(tx) + } + + fn get_latest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.inner_get_latest_block_number_hash(tx) + } + + fn storage_trie_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new( + cursor, + max_block_number, + Some(hashed_address), + )) + } + + fn account_trie_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new(cursor, max_block_number, None)) + } + + fn storage_hashed_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxStorageCursor::new( + cursor, + max_block_number, + hashed_address, + )) + } + + fn account_hashed_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxAccountCursor::new(cursor, max_block_number)) + } + + fn storage_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new( + cursor, + max_block_number, + Some(hashed_address), + )) + } + + fn account_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new(cursor, max_block_number, None)) + } + + fn storage_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxStorageCursor::new( + cursor, + max_block_number, + hashed_address, + )) + } + + fn account_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxAccountCursor::new(cursor, max_block_number)) + } + + fn store_trie_updates( + &self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + self.env + .update(|tx| self.store_trie_updates_append_only(tx, block_ref, block_state_diff))? + } + + fn fetch_trie_updates(&self, block_number: u64) -> MorphProofsStorageResult { + self.env.view(|tx| { + let mut change_set_cursor = tx.cursor_read::()?; + let (_, change_set) = change_set_cursor + .seek_exact(block_number)? + .ok_or(MorphProofsStorageError::NoChangeSetForBlock(block_number))?; + + let mut account_trie_cursor = tx.new_cursor::()?; + let mut storage_trie_cursor = tx.new_cursor::()?; + let mut hashed_account_cursor = tx.new_cursor::()?; + let mut hashed_storage_cursor = tx.new_cursor::()?; + + let mut trie_updates = TrieUpdates::default(); + for key in change_set.account_trie_keys { + let entry = + match account_trie_cursor.seek_by_key_subkey(key.clone(), block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingAccountTrieHistory( + key.0, + block_number, + )); + } + }; + + if let Some(value) = entry { + trie_updates.account_nodes.insert(key.0, value); + } else { + trie_updates.removed_nodes.insert(key.0); + } + } + + for key in change_set.storage_trie_keys { + let entry = + match storage_trie_cursor.seek_by_key_subkey(key.clone(), block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingStorageTrieHistory( + key.hashed_address, + key.path.0, + block_number, + )); + } + }; + + let stu = trie_updates + .storage_tries + .entry(key.hashed_address) + .or_insert_with(StorageTrieUpdates::default); + + // handle is_deleted scenario + // Issue: https://github.com/op-rs/op-reth/issues/323 + if let Some(value) = entry { + stu.storage_nodes.insert(key.path.0, value); + } else { + stu.removed_nodes.insert(key.path.0); + } + } + + let mut post_state = + HashedPostState::with_capacity(change_set.hashed_account_keys.len()); + for key in change_set.hashed_account_keys { + let entry = match hashed_account_cursor.seek_by_key_subkey(key, block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingHashedAccountHistory( + key, + block_number, + )); + } + }; + + post_state.accounts.insert(key, entry); + } + + for key in change_set.hashed_storage_keys { + let entry = + match hashed_storage_cursor.seek_by_key_subkey(key.clone(), block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingHashedStorageHistory { + hashed_address: key.hashed_address, + hashed_storage_key: key.hashed_storage_key, + block_number, + }); + } + }; + + let hs = post_state.storages.entry(key.hashed_address).or_default(); + + // handle wiped storage scenario + // Issue: https://github.com/op-rs/op-reth/issues/323 + if let Some(value) = entry { + hs.storage.insert(key.hashed_storage_key, value.0); + } else { + hs.storage.insert(key.hashed_storage_key, U256::ZERO); + } + } + + Ok(BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }) + })? + } + + /// Update the initial state with the provided diff. + /// Prune all historical trie data till `new_earliest_block_number` (inclusive) using + /// the [`BlockChangeSet`] index. + /// + /// Arguments: + /// - `new_earliest_block_ref`: The new earliest block reference (with parent hash). + /// - `diff`: The state diff to apply to the initial state (block 0). This diff represents all + /// the changes from the old earliest block to the new earliest block (inclusive). + fn prune_earliest_state( + &self, + new_earliest_block_ref: BlockWithParent, + ) -> MorphProofsStorageResult { + let target_block = new_earliest_block_ref.block.number; + + // --- PHASE 1: READ (Calculate Deletions) --- + let plan = self.calculate_prune_plan(target_block)?; + let Some(plan) = plan else { + return Ok(WriteCounts::default()); + }; + + // --- PHASE 2: WRITE (Execute Deletions) --- + self.env.update(|tx| { + // 1. Execute Sparse Deletions and track actual deleted rows + let acc_deleted = + self.prune_history_preceding::(tx, plan.acc_survivors)?; + + let st_deleted = + self.prune_history_preceding::(tx, plan.storage_survivors)?; + + let ha_deleted = self.prune_history_preceding::( + tx, + plan.hashed_acc_survivors, + )?; + + let hs_deleted = self.prune_history_preceding::( + tx, + plan.hashed_storage_survivors, + )?; + + let counts = WriteCounts { + account_trie_updates_written_total: acc_deleted, + storage_trie_updates_written_total: st_deleted, + hashed_accounts_written_total: ha_deleted, + hashed_storages_written_total: hs_deleted, + }; + + // 2. Delete ChangeSets + let range = (plan.earliest_block + 1)..=target_block; + let mut cs_cursor = tx.cursor_write::()?; + let mut walker = cs_cursor.walk_range(range)?; + while walker.next().is_some() { + walker.delete_current()?; + } + + // 3. Update Earliest Pointer + Self::inner_set_earliest_block_number( + tx, + target_block, + new_earliest_block_ref.block.hash, + )?; + + Ok(counts) + })? + } + + /// Unwind the historical state to `unwind_upto_block` (inclusive), deleting all history + /// starting from provided block. Also updates the `ProofWindow::LatestBlock` to parent of + /// `unwind_upto_block`. + fn unwind_history(&self, to: BlockWithParent) -> MorphProofsStorageResult<()> { + let history_to_delete = self + .env + .view(|tx| self.collect_history_ranged(tx, to.block.number..))??; + + self.env.update(|tx| { + let proof_window = match self.inner_get_proof_window(tx)? { + Some(pw) => pw, + None => return Ok(()), // Nothing to unwind + }; + + if to.block.number > proof_window.latest.number { + return Ok(()); // Nothing to unwind + } + + if to.block.number <= proof_window.earliest.number { + return Err(MorphProofsStorageError::UnwindBeyondEarliest { + unwind_block_number: to.block.number, + earliest_block_number: proof_window.earliest.number, + }); + } + + self.delete_history_ranged(tx, to.block.number.., history_to_delete)?; + + let new_latest_block = + BlockNumberHash::new(to.block.number.saturating_sub(1), to.parent); + + // Update proof window's Latest block + Self::inner_set_latest_block_number( + tx, + new_latest_block.number(), + *new_latest_block.hash(), + )?; + + Ok(()) + })? + } + + fn replace_updates( + &self, + latest_common_block: BlockNumHash, + mut blocks_to_add: Vec<(BlockWithParent, BlockStateDiff)>, + ) -> MorphProofsStorageResult<()> { + // Sort the vec list by block number + blocks_to_add.sort_unstable_by_key(|(bwp, _)| bwp.block.number); + + let history_to_delete = self + .env + .view(|tx| self.collect_history_ranged(tx, latest_common_block.number + 1..))??; + + self.env.update(|tx| { + // Remove the old history + self.delete_history_ranged(tx, latest_common_block.number + 1.., history_to_delete)?; + + // Update the ProofWindow Latest Block to latest_common_block so we can perform + // `store_trie_updates_append_only`. + Self::inner_set_latest_block_number( + tx, + latest_common_block.number, + latest_common_block.hash, + )?; + + // Apply the new history + for (block_with_parent, diff) in blocks_to_add { + self.store_trie_updates_append_only(tx, block_with_parent, diff)?; + } + Ok(()) + })? + } + + fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + self.set_earliest_block_number_hash(block_number, hash) + } +} + +impl MorphProofsBatchStore for MdbxProofsStorage { + type BatchSession<'a> + = MdbxBatchSession<'a> + where + Self: 'a; + + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult, + { + let tx = self.env.tx_mut()?; + let mut session = MdbxBatchSession::new(self, tx); + match f(&mut session) { + Ok(result) => { + session.commit()?; + Ok(result) + } + Err(err) => Err(err), + } + } +} + +impl MorphProofsInitialStateStore for MdbxProofsStorage { + fn initial_state_anchor(&self) -> MorphProofsStorageResult { + // 1) NotStarted: no anchor row + let Some(block) = self.get_initial_state_anchor()? else { + return Ok(InitialStateAnchor::default()); + }; + + // 2) Completed: anchor exists + earliest is set + let completed = self.get_earliest_block_number()?.is_some(); + + // 3) InProgress / Completed: populate details + Ok(InitialStateAnchor { + block: Some(block), + status: if completed { + InitialStateStatus::Completed + } else { + InitialStateStatus::InProgress + }, + latest_account_trie_key: self.get_latest_key::()?, + latest_storage_trie_key: self.get_latest_key::()?, + latest_hashed_account_key: self.get_latest_key::()?, + latest_hashed_storage_key: self.get_latest_key::()?, + }) + } + + fn set_initial_state_anchor(&self, anchor: BlockNumHash) -> MorphProofsStorageResult<()> { + self.env.update(|tx| { + let mut cur = tx.cursor_write::()?; + cur.insert(ProofWindowKey::InitialStateAnchor, &anchor.into())?; + Ok(()) + })? + } + + fn store_account_branches( + &self, + account_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut account_nodes = account_nodes; + if account_nodes.is_empty() { + return Ok(()); + } + + account_nodes.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch(tx, 0, account_nodes, true)?; + Ok(()) + })? + } + + fn store_storage_branches( + &self, + hashed_address: B256, + storage_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut storage_nodes = storage_nodes; + if storage_nodes.is_empty() { + return Ok(()); + } + + storage_nodes.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch( + tx, + 0, + storage_nodes + .into_iter() + .map(|(path, node)| (hashed_address, path, node)), + true, + )?; + Ok(()) + })? + } + + fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut accounts = accounts; + if accounts.is_empty() { + return Ok(()); + } + + // sort the accounts by key to ensure insertion is efficient + accounts.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch(tx, 0, accounts, true)?; + Ok(()) + })? + } + + fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> MorphProofsStorageResult<()> { + let mut storages = storages; + if storages.is_empty() { + return Ok(()); + } + + // sort the storages by key to ensure insertion is efficient + storages.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch( + tx, + 0, + storages + .into_iter() + .map(|(key, val)| (hashed_address, key, Some(StorageValue(val)))), + true, + )?; + Ok(()) + })? + } + + fn commit_initial_state(&self) -> MorphProofsStorageResult { + let anchor = self.get_initial_state_anchor()?.ok_or(NoBlocksFound)?; + self.set_earliest_block_number(anchor.number, anchor.hash)?; + Ok(anchor) + } +} + +#[cfg(test)] +mod tests { + use alloy_eips::NumHash; + use alloy_primitives::B256; + use reth_db::{ + DatabaseError, + cursor::DbDupCursorRO, + transaction::{DbTx, DbTxMut}, + }; + use reth_trie::{ + BranchNodeCompact, HashedPostStateSorted, HashedStorage, Nibbles, StoredNibbles, + updates::{StorageTrieUpdates, TrieUpdatesSorted}, + }; + use tempfile::TempDir; + + use super::*; + use crate::db::{ + StorageTrieKey, + models::{AccountTrieHistory, StorageTrieHistory}, + }; + + const B0: u64 = 0; + + #[test] + fn open_rejects_partial_identity_metadata_and_requires_manual_deletion() { + let dir = TempDir::new().unwrap(); + let env = init_db_for::<_, Tables>(dir.path(), DatabaseArguments::default()) + .expect("raw proof database"); + env.update(|tx| { + tx.put::( + ProofMetadataKey::ChainId, + ProofMetadataValue(2818u64.to_be_bytes().to_vec()), + ) + }) + .expect("write transaction") + .expect("write partial metadata"); + drop(env); + + let error = MdbxProofsStorage::open( + dir.path(), + ProofDbIdentity::new(2818, B256::repeat_byte(0x11)), + ) + .expect_err("partial identity metadata must fail closed"); + + assert!( + error.to_string().contains("manually delete"), + "operator must receive an explicit manual-deletion instruction: {error}" + ); + } + + #[test] + fn open_rejects_proof_data_without_identity_metadata() { + let identity = ProofDbIdentity::new(2818, B256::repeat_byte(0x11)); + + let proof_window_dir = TempDir::new().unwrap(); + let env = init_db_for::<_, Tables>(proof_window_dir.path(), DatabaseArguments::default()) + .expect("raw proof database"); + env.update(|tx| { + tx.put::( + ProofWindowKey::EarliestBlock, + BlockNumberHash::new(1, B256::repeat_byte(0x22)), + ) + }) + .expect("write transaction") + .expect("write proof window"); + drop(env); + + let error = MdbxProofsStorage::open(proof_window_dir.path(), identity) + .expect_err("proof-window data without identity metadata must fail closed"); + assert!( + error.to_string().contains("manually delete"), + "operator must receive an explicit manual-deletion instruction: {error}" + ); + + let history_dir = TempDir::new().unwrap(); + let env = init_db_for::<_, Tables>(history_dir.path(), DatabaseArguments::default()) + .expect("raw proof database"); + env.update(|tx| tx.put::(1, ChangeSet::default())) + .expect("write transaction") + .expect("write history change set"); + drop(env); + + let error = MdbxProofsStorage::open(history_dir.path(), identity) + .expect_err("history data without identity metadata must fail closed"); + assert!( + error.to_string().contains("manually delete"), + "operator must receive an explicit manual-deletion instruction: {error}" + ); + } + + #[test] + fn new_with_options_accepts_custom_max_read_transaction_duration() { + let dir = TempDir::new().unwrap(); + let options = MdbxProofsStorageOptions { + max_read_transaction_duration: Some(Duration::from_secs(30)), + }; + let store = MdbxProofsStorage::new_with_options(dir.path(), options).expect("env"); + + let _tx = store.env.tx().expect("ro tx"); + } + + #[test] + fn store_hashed_accounts_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0xAA; 32]); + let account = Account::default(); + store + .store_hashed_accounts(vec![(addr, Some(account))]) + .expect("write accounts"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let vv = cur.seek_by_key_subkey(addr, B0).expect("seek"); + let vv = vv.expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(account)); + } + + #[test] + fn store_hashed_accounts_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Unsorted input, mixed Some/None (deletion) + let a1 = B256::from([0x01; 32]); + let a2 = B256::from([0x02; 32]); + let a3 = B256::from([0x03; 32]); + let acc1 = Account { + nonce: 2, + balance: U256::from(1000u64), + ..Default::default() + }; + let acc3 = Account { + nonce: 1, + balance: U256::from(10000u64), + ..Default::default() + }; + + store + .store_hashed_accounts(vec![(a2, None), (a1, Some(acc1)), (a3, Some(acc3))]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur + .seek_by_key_subkey(a1, B0) + .expect("seek a1") + .expect("exists a1"); + assert_eq!(v1.block_number, B0); + assert_eq!(v1.value.0, Some(acc1)); + + let v2 = cur + .seek_by_key_subkey(a2, B0) + .expect("seek a2") + .expect("exists a2"); + assert_eq!(v2.block_number, B0); + assert!(v2.value.0.is_none(), "a2 is none"); + + let v3 = cur + .seek_by_key_subkey(a3, B0) + .expect("seek a3") + .expect("exists a3"); + assert_eq!(v3.block_number, B0); + assert_eq!(v3.value.0, Some(acc3)); + } + + #[test] + fn store_hashed_accounts_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Unsorted input, mixed Some/None (deletion) + let a1 = B256::from([0x01; 32]); + let a2 = B256::from([0x02; 32]); + let a3 = B256::from([0x03; 32]); + let a4 = B256::from([0x04; 32]); + let a5 = B256::from([0x05; 32]); + let acc1 = Account { + nonce: 2, + balance: U256::from(1000u64), + ..Default::default() + }; + let acc3 = Account { + nonce: 1, + balance: U256::from(10000u64), + ..Default::default() + }; + let acc4 = Account { + nonce: 5, + balance: U256::from(5000u64), + ..Default::default() + }; + let acc5 = Account { + nonce: 10, + balance: U256::from(20000u64), + ..Default::default() + }; + + { + store + .store_hashed_accounts(vec![(a2, None), (a1, Some(acc1)), (a4, Some(acc4))]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur + .seek_by_key_subkey(a1, B0) + .expect("seek a1") + .expect("exists a1"); + assert_eq!(v1.block_number, B0); + assert_eq!(v1.value.0, Some(acc1)); + + let v2 = cur + .seek_by_key_subkey(a2, B0) + .expect("seek a2") + .expect("exists a2"); + assert_eq!(v2.block_number, B0); + assert!(v2.value.0.is_none(), "a2 is none"); + + let v4 = cur + .seek_by_key_subkey(a4, B0) + .expect("seek a4") + .expect("exists a4"); + assert_eq!(v4.block_number, B0); + assert_eq!(v4.value.0, Some(acc4)); + } + + { + // Second call + store + .store_hashed_accounts(vec![(a5, Some(acc5)), (a3, Some(acc3))]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v3 = cur + .seek_by_key_subkey(a3, B0) + .expect("seek a3") + .expect("exists a3"); + assert_eq!(v3.block_number, B0); + assert_eq!(v3.value.0, Some(acc3)); + + let v5 = cur + .seek_by_key_subkey(a5, B0) + .expect("seek a5") + .expect("exists a5"); + assert_eq!(v5.block_number, B0); + assert_eq!(v5.value.0, Some(acc5)); + } + } + + #[test] + fn store_hashed_storages_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let slot = B256::from([0x22; 32]); + let val = U256::from(0x1234u64); + + store + .store_hashed_storages(addr, vec![(slot, val)]) + .expect("write storage"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let key = HashedStorageKey::new(addr, slot); + let vv = cur.seek_by_key_subkey(key, B0).expect("seek"); + let vv = vv.expect("entry exists"); + + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, val); + } + + #[test] + fn store_hashed_storages_multiple_slots_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let s1 = B256::from([0x01; 32]); + let v1 = U256::from(1u64); + let s2 = B256::from([0x02; 32]); + let v2 = U256::from(2u64); + let s3 = B256::from([0x03; 32]); + let v3 = U256::from(3u64); + + store + .store_hashed_storages(addr, vec![(s2, v2), (s1, v1), (s3, v3)]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s1, v1), (s2, v2), (s3, v3)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + + #[test] + fn store_hashed_storages_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let s1 = B256::from([0x01; 32]); + let v1 = U256::from(1u64); + let s2 = B256::from([0x02; 32]); + let v2 = U256::from(2u64); + let s3 = B256::from([0x03; 32]); + let v3 = U256::from(3u64); + let s4 = B256::from([0x04; 32]); + let v4 = U256::from(4u64); + let s5 = B256::from([0x05; 32]); + let v5 = U256::from(5u64); + + { + store + .store_hashed_storages(addr, vec![(s2, v2), (s1, v1), (s5, v5)]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s1, v1), (s2, v2), (s5, v5)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + + { + // Second call + store + .store_hashed_storages(addr, vec![(s4, v4), (s3, v3)]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s4, v4), (s3, v3)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + } + + #[test] + fn test_store_account_branches_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let nibble = Nibbles::from_nibbles_unchecked([0x12, 0x34]); + let branch_node = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let updates = vec![(nibble, Some(branch_node.clone()))]; + + store.store_account_branches(updates).expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + let vv = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(branch_node)); + } + + #[test] + fn test_store_account_branches_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + let updates = vec![(n2, None), (n1, Some(b1)), (n3, Some(b3))]; + store + .store_account_branches(updates.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + #[test] + fn store_account_branches_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n4 = Nibbles::from_nibbles_unchecked([0x04]); + let b4 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n5 = Nibbles::from_nibbles_unchecked([0x05]); + let b5 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + { + let updates1 = vec![(n2, None), (n1, Some(b1)), (n4, Some(b4))]; + store + .store_account_branches(updates1.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates1 { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + { + // Second call + let updates2 = vec![(n5, Some(b5)), (n3, Some(b3))]; + store + .store_account_branches(updates2.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates2 { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + } + + #[test] + fn test_store_storage_branches_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let nibble = Nibbles::from_nibbles_unchecked([0x12, 0x34]); + let branch_node = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let items = vec![(nibble, Some(branch_node.clone()))]; + + store + .store_storage_branches(hashed_address, items) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(branch_node)); + } + + #[test] + fn store_storage_branches_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + let items = vec![(n2, None), (n1, Some(b1)), (n3, Some(b3))]; + store + .store_storage_branches(hashed_address, items.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + #[test] + fn store_storage_branches_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n4 = Nibbles::from_nibbles_unchecked([0x04]); + let b4 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n5 = Nibbles::from_nibbles_unchecked([0x05]); + let b5 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + { + let items1 = vec![(n2, None), (n1, Some(b1)), (n5, Some(b5))]; + store + .store_storage_branches(hashed_address, items1.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items1 { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + { + // Second call + let items2 = vec![(n4, Some(b4)), (n3, Some(b3))]; + store + .store_storage_branches(hashed_address, items2.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items2 { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + } + + #[test] + fn test_store_trie_updates_comprehensive() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Sample block number + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(42, B256::ZERO)); + + // Sample addresses and keys + let addr1 = B256::from([0x11; 32]); + let addr2 = B256::from([0x22; 32]); + let slot1 = B256::from([0xA1; 32]); + let slot2 = B256::from([0xA2; 32]); + + // Sample accounts + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + + // Sample storage values + let val1 = U256::from(1234u64); + let val2 = U256::from(5678u64); + + // Sample trie paths + let account_path1 = Nibbles::from_nibbles_unchecked(vec![0, 1, 2, 3]); + let account_path2 = Nibbles::from_nibbles_unchecked(vec![4, 5, 6, 7]); + let removed_account_path = Nibbles::from_nibbles_unchecked(vec![7, 8, 9]); + + let account_node1 = BranchNodeCompact::default(); + let account_node2 = BranchNodeCompact::default(); + + let storage_path1 = Nibbles::from_nibbles_unchecked(vec![1, 2, 3, 4]); + let storage_path2 = Nibbles::from_nibbles_unchecked(vec![8, 9, 0, 1]); + + let storage_node1 = BranchNodeCompact::default(); + let storage_node2 = BranchNodeCompact::default(); + + // Construct test BlockStateDiff + let mut block_state_diff_trie_updates = TrieUpdates::default(); + let mut block_state_diff_post_state = HashedPostState::default(); + + // Add account trie nodes + block_state_diff_trie_updates + .account_nodes + .insert(account_path1, account_node1); + block_state_diff_trie_updates + .account_nodes + .insert(account_path2, account_node2); + block_state_diff_trie_updates + .removed_nodes + .insert(removed_account_path); + + // Add storage trie nodes for two addresses + let mut storage_nodes1 = StorageTrieUpdates::default(); + storage_nodes1 + .storage_nodes + .insert(storage_path1, storage_node1); + block_state_diff_trie_updates + .storage_tries + .insert(addr1, storage_nodes1); + + let mut storage_nodes2 = StorageTrieUpdates::default(); + storage_nodes2 + .storage_nodes + .insert(storage_path2, storage_node2); + block_state_diff_trie_updates + .storage_tries + .insert(addr2, storage_nodes2); + + // Add hashed accounts (one Some, one None) + block_state_diff_post_state + .accounts + .insert(addr1, Some(acc1)); + block_state_diff_post_state.accounts.insert(addr2, None); // Deletion + + // Add storage slots for both addresses + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, val1); + block_state_diff_post_state.storages.insert(addr1, storage1); + + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot2, val2); + block_state_diff_post_state.storages.insert(addr2, storage2); + + // Store everything + let block_state_diff = BlockStateDiff { + sorted_trie_updates: block_state_diff_trie_updates.into_sorted(), + sorted_post_state: block_state_diff_post_state.into_sorted(), + }; + store + .store_trie_updates(BLOCK, block_state_diff) + .expect("store"); + + // Verify account trie nodes + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check first node + let vv1 = cur + .seek_by_key_subkey(account_path1.into(), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + assert!(vv1.value.0.is_some()); + + // Check second node + let vv2 = cur + .seek_by_key_subkey(account_path2.into(), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + assert!(vv2.value.0.is_some()); + + // Check removed node + let vv3 = cur + .seek_by_key_subkey(removed_account_path.into(), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv3.block_number, BLOCK.block.number); + assert!(vv3.value.0.is_none(), "Expected node deletion"); + } + + // Verify storage trie nodes + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check node for addr1 + let key1 = StorageTrieKey::new(addr1, storage_path1.into()); + let vv1 = cur + .seek_by_key_subkey(key1, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + assert!(vv1.value.0.is_some()); + + // Check node for addr2 + let key2 = StorageTrieKey::new(addr2, storage_path2.into()); + let vv2 = cur + .seek_by_key_subkey(key2, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + assert!(vv2.value.0.is_some()); + } + + // Verify hashed accounts + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check account1 (exists) + let vv1 = cur + .seek_by_key_subkey(addr1, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + assert_eq!(vv1.value.0, Some(acc1)); + + // Check account2 (deletion) + let vv2 = cur + .seek_by_key_subkey(addr2, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + assert!(vv2.value.0.is_none(), "Expected account deletion"); + } + + // Verify hashed storages + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check storage for addr1 + let key1 = HashedStorageKey::new(addr1, slot1); + let vv1 = cur + .seek_by_key_subkey(key1, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + let inner1 = vv1.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner1.0, val1); + + // Check storage for addr2 + let key2 = HashedStorageKey::new(addr2, slot2); + let vv2 = cur + .seek_by_key_subkey(key2, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + let inner2 = vv2.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner2.0, val2); + } + + // Verify BlockChangeSet entries + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let entries: Vec<_> = cur.walk(Some(BLOCK.block.number)).expect("walk").collect(); + assert_eq!(entries.len(), 1, "Expected 1 BlockChangeSet entry"); + } + + // check the latest block number in proof window + { + let tx = store.env.tx().expect("tx"); + let mut proof_window_cursor = tx.new_cursor::().expect("cursor"); + let latest_block = proof_window_cursor + .seek(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("exists"); + assert_eq!(latest_block.1.number(), BLOCK.block.number); + assert_eq!(*latest_block.1.hash(), BLOCK.block.hash); + } + } + + #[test] + fn store_trie_updates_out_of_order_rejects() { + let dir = tempfile::TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // set latest to some hash H1 + let existing_block = BlockWithParent::new(B256::random(), NumHash::new(1, B256::random())); + store + .set_earliest_block_number(existing_block.block.number, existing_block.block.hash) + .expect("set"); + + // incoming block whose parent != existing latest + let bad_parent = B256::from([0xFF; 32]); + let bad_block = BlockWithParent::new(bad_parent, NumHash::new(2, B256::ZERO)); + let diff = BlockStateDiff::default(); + + let res = store.store_trie_updates(bad_block, diff); + assert!(matches!( + res, + Err(MorphProofsStorageError::OutOfOrder { .. }) + )); + // verify nothing written: proof window still unchanged + let latest = store.get_latest_block_number().expect("get latest"); + assert_eq!(latest.unwrap().1, existing_block.block.hash); + } + + #[test] + fn store_trie_updates_multiple_blocks_append_versions() { + let dir = tempfile::TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x21; 32]); + // block A (parent = ZERO) + let block_a = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut diff_a_post_state = HashedPostState::default(); + diff_a_post_state + .accounts + .insert(addr, Some(Account::default())); + + let diff_a = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_a_post_state.into_sorted(), + }; + store.store_trie_updates(block_a, diff_a).expect("store A"); + + // block B (parent = hash of A) + let block_b = BlockWithParent::new(block_a.block.hash, NumHash::new(2, B256::random())); + let mut diff_b_post_state = HashedPostState::default(); + diff_b_post_state.accounts.insert( + addr, + Some(Account { + nonce: 5, + ..Default::default() + }), + ); + + let diff_b = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_b_post_state.into_sorted(), + }; + store.store_trie_updates(block_b, diff_b).expect("store B"); + + // verify we can retrieve entries for both block numbers + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let v_a = cur + .seek_by_key_subkey(addr, block_a.block.number) + .expect("seek") + .expect("exists"); + let v_b = cur + .seek_by_key_subkey(addr, block_b.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(v_a.block_number, block_a.block.number); + assert_eq!(v_b.block_number, block_b.block.number); + } + + #[test] + fn test_store_trie_updates_empty_collections() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(42, B256::ZERO)); + + // Create BlockStateDiff with empty collections + let block_state_diff = BlockStateDiff::default(); + + // This should work without errors + store + .store_trie_updates(BLOCK, block_state_diff) + .expect("store"); + + // Verify nothing was written (should be empty) + let tx = store.env.tx().expect("tx"); + + let mut cur1 = tx.new_cursor::().expect("cursor"); + assert!( + cur1.next_dup_val().expect("first").is_none(), + "Account trie should be empty" + ); + + let mut cur2 = tx.new_cursor::().expect("cursor"); + assert!( + cur2.next_dup_val().expect("first").is_none(), + "Storage trie should be empty" + ); + + let mut cur3 = tx.new_cursor::().expect("cursor"); + assert!( + cur3.next_dup_val().expect("first").is_none(), + "Hashed accounts should be empty" + ); + + let mut cur4 = tx.new_cursor::().expect("cursor"); + assert!( + cur4.next_dup_val().expect("first").is_none(), + "Hashed storage should be empty" + ); + + let mut cur5 = tx.new_cursor::().expect("cursor"); + assert!( + cur5.next().expect("first").is_some(), + "Pruning index SHOULD populate the change set even for empty diffs" + ); + } + + #[test] + fn fetch_trie_updates_missing_changeset_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let res = store.fetch_trie_updates(99); + assert!(matches!( + res, + Err(MorphProofsStorageError::NoChangeSetForBlock(99)) + )); + } + + #[test] + fn fetch_trie_updates_empty_changeset() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let diff = BlockStateDiff::default(); + + store.store_trie_updates(block, diff).expect("store"); + let got = store.fetch_trie_updates(1).expect("fetch"); + assert!(got.sorted_trie_updates.account_nodes_ref().is_empty()); + assert!(got.sorted_trie_updates.storage_tries_ref().is_empty()); + assert!(got.sorted_post_state.accounts.is_empty()); + assert!(got.sorted_post_state.storages.is_empty()); + } + + #[test] + fn fetch_trie_updates_missing_account_history_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references StoredNibbles for account key + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + account_trie_keys: vec![StoredNibbles::default()], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingAccountTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_account_history_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert account history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut acc_cur = tx.cursor_write::().unwrap(); + acc_cur + .insert( + StoredNibbles::from(Nibbles::from_nibbles_unchecked([0x1])), + &VersionedValue::new(2, MaybeDeleted(Some(BranchNodeCompact::default()))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + account_trie_keys: vec![StoredNibbles::from(Nibbles::from_nibbles_unchecked( + [0x1], + ))], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingAccountTrieHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingAccountTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_missing_storage_history_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references StorageTrieKey for storage trie + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + storage_trie_keys: vec![StorageTrieKey::new( + B256::from([0u8; 32]), + StoredNibbles::default(), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingStorageTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_storage_history_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert storage history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut stor_cur = tx.cursor_write::().unwrap(); + stor_cur + .insert( + StorageTrieKey::new( + B256::from([0u8; 32]), + StoredNibbles::from(Nibbles::from_nibbles_unchecked([0x1])), + ), + &VersionedValue::new(2, MaybeDeleted(Some(BranchNodeCompact::default()))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + storage_trie_keys: vec![StorageTrieKey::new( + B256::from([0u8; 32]), + StoredNibbles::from(Nibbles::from_nibbles_unchecked([0x1])), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingStorageTrieHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingStorageTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_missing_hashed_account_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references hashed account address + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_account_keys: vec![B256::from([0u8; 32])], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedAccountHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_hashed_account_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert hashed account history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut acc_cur = tx.cursor_write::().unwrap(); + acc_cur + .insert( + B256::from([0u8; 32]), + &VersionedValue::new(2, MaybeDeleted(Some(Account::default()))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_account_keys: vec![B256::from([0u8; 32])], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingHashedAccountHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedAccountHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_missing_hashed_storage_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references hashed storage key + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_storage_keys: vec![HashedStorageKey::new( + B256::from([0u8; 32]), + B256::from([0u8; 32]), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedStorageHistory { .. }) + )); + } + + #[test] + fn fetch_trie_updates_hashed_storage_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert hashed storage history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut stor_cur = tx.cursor_write::().unwrap(); + stor_cur + .insert( + HashedStorageKey::new(B256::from([0u8; 32]), B256::from([0u8; 32])), + &VersionedValue::new(2, MaybeDeleted(Some(StorageValue::new(U256::ZERO)))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_storage_keys: vec![HashedStorageKey::new( + B256::from([0u8; 32]), + B256::from([0u8; 32]), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingHashedStorageHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedStorageHistory { .. }) + )); + } + + #[test] + fn fetch_trie_updates_basic() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Build a block with mixed changes (accounts, trie nodes, hashed storages) + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + + // prepare data + let addr1 = B256::from([0x11; 32]); + let addr2 = B256::from([0x22; 32]); + let slot1 = B256::from([0xA1; 32]); + let slot2 = B256::from([0xA2; 32]); + + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + + let val1 = U256::from(1234u64); + let val2 = U256::from(5678u64); + + let account_path1 = Nibbles::from_nibbles_unchecked(vec![0, 1, 2, 3]); + let account_path2 = Nibbles::from_nibbles_unchecked(vec![4, 5, 6, 7]); + let account_node1 = BranchNodeCompact { + root_hash: Some(B256::random()), + ..Default::default() + }; + let account_node2 = BranchNodeCompact { + root_hash: Some(B256::random()), + ..Default::default() + }; + + let storage_path1 = Nibbles::from_nibbles_unchecked(vec![1, 2, 3, 4]); + let storage_node1 = BranchNodeCompact { + root_hash: Some(B256::random()), + ..Default::default() + }; + + // Construct BlockStateDiff + let mut block_state_diff_trie_updates = TrieUpdates::default(); + block_state_diff_trie_updates + .account_nodes + .insert(account_path1, account_node1); + block_state_diff_trie_updates + .account_nodes + .insert(account_path2, account_node2); + // storage trie for addr1 + let mut storage_nodes1 = StorageTrieUpdates::default(); + storage_nodes1 + .storage_nodes + .insert(storage_path1, storage_node1); + block_state_diff_trie_updates + .storage_tries + .insert(addr1, storage_nodes1); + + // hashed accounts: addr1 -> Some, addr2 -> None + let mut block_state_diff_post_state = HashedPostState::default(); + block_state_diff_post_state + .accounts + .insert(addr1, Some(acc1)); + block_state_diff_post_state.accounts.insert(addr2, None); + + // hashed storages + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, val1); + block_state_diff_post_state.storages.insert(addr1, storage1); + + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot2, val2); + block_state_diff_post_state.storages.insert(addr2, storage2); + + // store then fetch + let block_state_diff = BlockStateDiff { + sorted_trie_updates: block_state_diff_trie_updates.into_sorted(), + sorted_post_state: block_state_diff_post_state.into_sorted(), + }; + store + .store_trie_updates(block, block_state_diff.clone()) + .expect("store"); + let got = store.fetch_trie_updates(1).expect("fetch"); + + // verify trie updates + assert_eq!( + got.sorted_trie_updates.account_nodes_ref(), + block_state_diff.sorted_trie_updates.account_nodes_ref(), + ); + assert_eq!( + got.sorted_trie_updates.storage_tries_ref(), + block_state_diff.sorted_trie_updates.storage_tries_ref(), + ); + + // verify post state + assert_eq!( + got.sorted_post_state.accounts, + block_state_diff.sorted_post_state.accounts + ); + assert_eq!( + got.sorted_post_state.storages, + block_state_diff.sorted_post_state.storages + ); + } + + #[test] + fn test_prune_earliest_state_single_entry() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert a single entry to be pruned + let addr = B256::random(); + let mut state_diff_post_state = HashedPostState::default(); + state_diff_post_state + .accounts + .insert(addr, Some(Account::default())); + let state_diff = BlockStateDiff { + sorted_post_state: state_diff_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block, state_diff).unwrap(); + + // Prune the entry - pass empty diff since we're just removing data + let next_block = BlockWithParent::new(block.block.hash, NumHash::new(2, B256::random())); + store.prune_earliest_state(next_block).unwrap(); + + // Verify the entry was pruned + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // The survivor at block 1 should remain + let val = cur + .seek_by_key_subkey(addr, block.block.number) + .unwrap() + .expect("Survivor should exist"); + assert_eq!(val.block_number, 1); + + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!( + pruning_cur + .seek_exact(block.block.number) + .unwrap() + .is_none() + ); + + // Verify earliest block was updated + let earliest = store.get_earliest_block_number().unwrap(); + assert_eq!(earliest, Some((2, next_block.block.hash))); + } + + #[test] + fn test_prune_earliest_state_multiple_entries_same_block() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert multiple entries for the same block + let addr1 = B256::random(); + let addr2 = B256::random(); + let mut state_diff_post_state = HashedPostState::default(); + state_diff_post_state + .accounts + .insert(addr1, Some(Account::default())); + state_diff_post_state + .accounts + .insert(addr2, Some(Account::default())); + let state_diff = BlockStateDiff { + sorted_post_state: state_diff_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block, state_diff).unwrap(); + + // Prune the entries + let next_block = BlockWithParent::new(block.block.hash, NumHash::new(2, B256::random())); + store.prune_earliest_state(next_block).unwrap(); + + // Verify the entries were pruned + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // All entries should survive at block 1 + let v1 = cur + .seek_by_key_subkey(addr1, block.block.number) + .unwrap() + .expect("addr1 survivor"); + assert_eq!(v1.block_number, 1); + let v2 = cur + .seek_by_key_subkey(addr2, block.block.number) + .unwrap() + .expect("addr2 survivor"); + assert_eq!(v2.block_number, 1); + + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!( + pruning_cur + .seek_exact(block.block.number) + .unwrap() + .is_none() + ); + } + + #[test] + fn test_prune_earliest_state_multiple_blocks() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert entries for multiple blocks + let addr1 = B256::random(); + let addr2 = B256::random(); + let mut state_diff1_post_state = HashedPostState::default(); + state_diff1_post_state + .accounts + .insert(addr1, Some(Account::default())); + let state_diff1 = BlockStateDiff { + sorted_post_state: state_diff1_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, state_diff1).unwrap(); + + let mut state_diff2_post_state = HashedPostState::default(); + state_diff2_post_state + .accounts + .insert(addr2, Some(Account::default())); + let state_diff2 = BlockStateDiff { + sorted_post_state: state_diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, state_diff2).unwrap(); + + // Prune up to block 3 (should remove blocks 1 and 2) + store.prune_earliest_state(block_3).unwrap(); + + // Verify the entries were pruned + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // addr1 survivor at block 1 must exist + let v1 = cur + .seek_by_key_subkey(addr1, 1) + .unwrap() + .expect("addr1 survivor"); + assert_eq!(v1.block_number, 1); + + // addr2 survivor at block 2 must exist + let v2 = cur + .seek_by_key_subkey(addr2, 2) + .unwrap() + .expect("addr2 survivor"); + assert_eq!(v2.block_number, 2); + + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!(pruning_cur.seek_exact(1).unwrap().is_none()); + assert!(pruning_cur.seek_exact(2).unwrap().is_none()); + } + + #[test] + fn test_prune_earliest_state_no_op() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(1, B256::random()).unwrap(); + + // Attempt to prune with a new earliest block that is not newer + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let block_0 = BlockWithParent::new(B256::ZERO, NumHash::new(0, B256::random())); + store.prune_earliest_state(block_1).unwrap(); + store.prune_earliest_state(block_0).unwrap(); + + // Nothing should have been pruned, this call should not panic or error + } + + #[test] + fn test_prune_earliest_state_no_entries_to_prune() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(1, B256::random()).unwrap(); + + // Prune a range where no entries exist + let block_10 = BlockWithParent::new(B256::ZERO, NumHash::new(10, B256::random())); + store.prune_earliest_state(block_10).unwrap(); + + // Nothing should have been pruned, this call should not panic or error + } + + #[test] + fn test_prune_earliest_state_with_diff_insertion() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert entries for blocks 1 and 2 + let addr1 = B256::random(); + let addr2 = B256::random(); + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut state_diff1_post_state = HashedPostState::default(); + state_diff1_post_state.accounts.insert(addr1, Some(acc1)); + let state_diff1 = BlockStateDiff { + sorted_post_state: state_diff1_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, state_diff1).unwrap(); + + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let mut state_diff2_post_state = HashedPostState::default(); + state_diff2_post_state.accounts.insert(addr2, Some(acc2)); + let state_diff2 = BlockStateDiff { + sorted_post_state: state_diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, state_diff2).unwrap(); + + // Now prune to block 3 + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.prune_earliest_state(block_3).unwrap(); + + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // Verify that blocks 1 and 2 entries were NOT pruned (they are survivors for their + // respective keys) + let vv1 = cur + .seek_by_key_subkey(addr1, 0) + .unwrap() + .expect("addr1 should survive"); + assert_eq!(vv1.block_number, 1); + assert_eq!(vv1.value.0, Some(acc1)); + + let vv2 = cur + .seek_by_key_subkey(addr2, 0) + .unwrap() + .expect("addr2 should survive"); + assert_eq!(vv2.block_number, 2); + assert_eq!(vv2.value.0, Some(acc2)); + + // Verify change sets for blocks 1 and 2 were removed + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!(pruning_cur.seek_exact(1).unwrap().is_none()); + assert!(pruning_cur.seek_exact(2).unwrap().is_none()); + + // Verify earliest block was updated + let earliest = store.get_earliest_block_number().unwrap(); + assert_eq!(earliest, Some((3, block_3.block.hash))); + } + + #[test] + fn test_prune_earliest_state_with_removed_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Create some trie nodes in blocks 1, 2, 3 + let path1 = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let path2 = Nibbles::from_nibbles_unchecked([0x03, 0x04]); + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let node2 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut diff1_trie_updates = TrieUpdates::default(); + diff1_trie_updates.account_nodes.insert(path1, node1); + let diff1 = BlockStateDiff { + sorted_trie_updates: diff1_trie_updates.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, diff1).unwrap(); + + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let mut diff2_trie_updates = TrieUpdates::default(); + diff2_trie_updates + .account_nodes + .insert(path2, node2.clone()); + let diff2 = BlockStateDiff { + sorted_trie_updates: diff2_trie_updates.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, diff2).unwrap(); + + // In block 3, path1 is deleted (stored as None in the database) + // This happens when we store trie updates with path1 mapped to None + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + // Simulate storing a deletion by directly writing to DB + store + .env + .update(|tx| { + let mut cursor = tx.new_cursor::()?; + let vv = VersionedValue { + block_number: 3, + value: MaybeDeleted(None), + }; + cursor.upsert(StoredNibbles::from(path1), &vv)?; + + // Record in change set + let mut change_set_cursor = tx.new_cursor::()?; + change_set_cursor.upsert( + 3, + &ChangeSet { + account_trie_keys: vec![StoredNibbles::from(path1)], + storage_trie_keys: vec![], + hashed_account_keys: vec![], + hashed_storage_keys: vec![], + }, + )?; + + // Update proof window + let mut proof_window_cursor = tx.new_cursor::()?; + proof_window_cursor.upsert( + ProofWindowKey::LatestBlock, + &BlockNumberHash::new(3, block_3.block.hash), + )?; + Ok::<(), DatabaseError>(()) + }) + .unwrap() + .unwrap(); + + // Now prune to block 5, with the new initial state: + // - path1 should be in removed_nodes (it was deleted in block 3) + // - path2 should be included with its value (it still exists from block 2) + let block_5 = BlockWithParent::new(B256::random(), NumHash::new(5, B256::random())); + store.prune_earliest_state(block_5).unwrap(); + + // Verify that all entries for path1 before block 5 were removed + let tx = store.env.tx().unwrap(); + let mut cur = tx.cursor_dup_read::().unwrap(); + + // path1 at block 1 should be gone; seeking 1 finds survivor (tombstone at 3) + if let Some(v) = cur + .seek_by_key_subkey(StoredNibbles::from(path1), 1) + .unwrap() + { + assert!(v.block_number >= 3, "path1 at block 1 should be pruned"); + } + + // path1 at block 1 should be gone. + // path1 survivor at block 3 (tombstone) should ALSO be gone now (optimization). + // So seeking for path1 should return None. + assert!( + cur.seek_by_key_subkey(StoredNibbles::from(path1), 0) + .unwrap() + .is_none(), + "path1 should be completely removed including tombstone" + ); + + // path2 entries should be pruned (blocks < 5) + // Survivor for path2 is at block 2. + let v2 = cur + .seek_by_key_subkey(StoredNibbles::from(path2), 0) + .unwrap() + .expect("path2 survivor"); + assert_eq!(v2.block_number, 2); + assert_eq!(v2.value.0, Some(node2)); + } + + #[test] + fn test_prune_earliest_state_overlapping_keys() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Use overlapping key (addr1 updated in blocks 1 and 2) + let addr1 = B256::random(); + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut diff1_post_state = HashedPostState::default(); + diff1_post_state.accounts.insert(addr1, Some(acc1)); + let diff1 = BlockStateDiff { + sorted_post_state: diff1_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, diff1).unwrap(); + + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let mut diff2_post_state = HashedPostState::default(); + diff2_post_state.accounts.insert(addr1, Some(acc2)); + let diff2 = BlockStateDiff { + sorted_post_state: diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, diff2).unwrap(); + + // Prune to block 3 + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.prune_earliest_state(block_3).unwrap(); + + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // Verify survivor (acc2 at block 2) remains + let vv1 = cur + .seek_by_key_subkey(addr1, 0) + .unwrap() + .expect("addr1 should having surviving state"); + assert_eq!(vv1.block_number, 2); + assert_eq!(vv1.value.0, Some(acc2)); + } + + #[test] + fn test_prune_earliest_state_comprehensive() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Setup complex scenario with accounts, storage, and trie nodes + let addr1 = B256::random(); + let slot1 = B256::random(); + let path1 = Nibbles::from_nibbles_unchecked([0x01]); + let storage_path1 = Nibbles::from_nibbles_unchecked([0x03]); + + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let storage_node1 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + + // Block 1: Insert account, trie node, and storage for addr1 + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + + let mut diff1_trie_updates = TrieUpdates::default(); + let mut diff1_post_state = HashedPostState::default(); + + diff1_post_state.accounts.insert(addr1, Some(acc1)); + diff1_trie_updates + .account_nodes + .insert(path1, node1.clone()); + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, U256::from(1234)); + diff1_post_state.storages.insert(addr1, storage1.clone()); + let mut storage_updates1 = StorageTrieUpdates::default(); + storage_updates1 + .storage_nodes + .insert(storage_path1, storage_node1.clone()); + diff1_trie_updates + .storage_tries + .insert(addr1, storage_updates1.clone()); + + let diff_1 = BlockStateDiff { + sorted_post_state: diff1_post_state.into_sorted(), + sorted_trie_updates: diff1_trie_updates.into_sorted(), + }; + store.store_trie_updates(block_1, diff_1).unwrap(); + + // Block 2: Update account (overwriting addr1) + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + + let mut diff2_post_state = HashedPostState::default(); + diff2_post_state.accounts.insert(addr1, Some(acc2)); + let diff2 = BlockStateDiff { + sorted_post_state: diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, diff2).unwrap(); + + // Prune to block 3 + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.prune_earliest_state(block_3).unwrap(); + + let tx = store.env.tx().unwrap(); + + // Verify account history - acc1 at block 1 pruned + let mut acc_cur = tx.new_cursor::().unwrap(); + // Survivor acc2 at block 2 remains + let acc_vv = acc_cur + .seek_by_key_subkey(addr1, 0) + .unwrap() + .expect("Survivor at block 2"); + assert_eq!(acc_vv.value.0, Some(acc2)); + assert_eq!(acc_vv.block_number, 2); + + // Verify account trie history - path1 at block 1 remains because it's the latest for that + // key + let mut trie_cur = tx.cursor_dup_read::().unwrap(); + let trie_vv = trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 0) + .unwrap() + .expect("Survivor at block 1"); + assert_eq!(trie_vv.value.0, Some(node1)); + assert_eq!(trie_vv.block_number, 1); + + // Verify storage history - slot1 at block 1 remains + let mut storage_cur = tx.new_cursor::().unwrap(); + let storage_key = HashedStorageKey::new(addr1, slot1); + let storage_vv = storage_cur + .seek_by_key_subkey(storage_key, 0) + .unwrap() + .expect("Survivor at block 1"); + assert_eq!(storage_vv.value.0.as_ref().unwrap().0, U256::from(1234)); + assert_eq!(storage_vv.block_number, 1); + + // Verify storage trie history + let mut storage_trie_cur = tx.cursor_dup_read::().unwrap(); + let storage_trie_key = StorageTrieKey::new(addr1, StoredNibbles::from(storage_path1)); + let storage_trie_vv = storage_trie_cur + .seek_by_key_subkey(storage_trie_key, 0) + .unwrap() + .expect("Survivor at block 1"); + assert_eq!(storage_trie_vv.value.0, Some(storage_node1)); + assert_eq!(storage_trie_vv.block_number, 1); + + // Verify change sets pruned + let mut change_cur = tx.new_cursor::().unwrap(); + assert!(change_cur.seek_exact(1).unwrap().is_none()); + assert!(change_cur.seek_exact(2).unwrap().is_none()); + } + + #[test] + fn test_prune_earliest_state_churn_create_delete_recreate() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + let addr = B256::random(); + + // Block 1: Create + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let acc1 = Account { + nonce: 1, + ..Default::default() + }; + let mut diff1 = HashedPostState::default(); + diff1.accounts.insert(addr, Some(acc1)); + store + .store_trie_updates( + b1, + BlockStateDiff { + sorted_post_state: diff1.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Block 2: Delete + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let mut diff2 = HashedPostState::default(); + diff2.accounts.insert(addr, None); + store + .store_trie_updates( + b2, + BlockStateDiff { + sorted_post_state: diff2.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Block 3: Recreate + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let acc3 = Account { + nonce: 3, + ..Default::default() + }; + let mut diff3 = HashedPostState::default(); + diff3.accounts.insert(addr, Some(acc3)); + store + .store_trie_updates( + b3, + BlockStateDiff { + sorted_post_state: diff3.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Prune to Block 3 + store.prune_earliest_state(b3).unwrap(); + + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // Block 1 (Older than 3) should be deleted. + // Since Block 3 exists, seek(1) will land on Block 3. + if let Some(val) = cur.seek_by_key_subkey(addr, 1).unwrap() { + assert!( + val.block_number >= 3, + "Block 1 should use be pruned, found {}", + val.block_number + ); + } + + // Block 2 (Older than 3) should be deleted + if let Some(val) = cur.seek_by_key_subkey(addr, 2).unwrap() { + assert!( + val.block_number >= 3, + "Block 2 should use be pruned, found {}", + val.block_number + ); + } + + // Block 3 (Survivor) should exist + let val = cur + .seek_by_key_subkey(addr, 3) + .unwrap() + .expect("Block 3 should survive"); + assert_eq!(val.block_number, 3); + assert_eq!(val.value.0, Some(acc3)); + } + + #[test] + fn test_prune_earliest_state_returns_correct_counts() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + let addr = B256::random(); + + // Block 1: Insert + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let acc1 = Account { + nonce: 1, + ..Default::default() + }; + let mut diff1 = HashedPostState::default(); + diff1.accounts.insert(addr, Some(acc1)); + store + .store_trie_updates( + b1, + BlockStateDiff { + sorted_post_state: diff1.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Block 2: Update + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let acc2 = Account { + nonce: 2, + ..Default::default() + }; + let mut diff2 = HashedPostState::default(); + diff2.accounts.insert(addr, Some(acc2)); + store + .store_trie_updates( + b2, + BlockStateDiff { + sorted_post_state: diff2.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Prune to Block 2. + // Survivor is at Block 2. + // Block 1 should be deleted. + // Count should be 1. + let counts = store.prune_earliest_state(b2).unwrap(); + + assert_eq!(counts.hashed_accounts_written_total, 1); + assert_eq!(counts.account_trie_updates_written_total, 0); + } + + #[test] + fn test_prune_earliest_state_empty_window_updates_pointer() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + let target = BlockWithParent::new(B256::random(), NumHash::new(5, B256::random())); + + // Prune empty + store.prune_earliest_state(target).unwrap(); + + let earliest = store.get_earliest_block_number().unwrap(); + assert_eq!(earliest, Some((5, target.block.hash))); + } + + #[test] + fn test_prune_earliest_state_uninitialized_guard() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Earliest not set + let target = BlockWithParent::new(B256::random(), NumHash::new(5, B256::random())); + + let counts = store.prune_earliest_state(target).unwrap(); + assert_eq!(counts, WriteCounts::default()); + + // Check earliest is still None + assert_eq!(store.get_earliest_block_number().unwrap(), None); + } + + #[test] + fn test_block_change_set_crud_operations() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let tx = store.env.tx_mut().expect("rw tx"); + let mut cursor = tx.cursor_write::().expect("cursor"); + + let block_1 = 42u64; + let block_2 = 43u64; + + let entry1 = ChangeSet { + account_trie_keys: vec![StoredNibbles::default()], + storage_trie_keys: vec![], + hashed_account_keys: vec![B256::ZERO], + hashed_storage_keys: vec![], + }; + let entry2 = ChangeSet { + account_trie_keys: vec![], + storage_trie_keys: vec![StorageTrieKey::new(B256::ZERO, StoredNibbles::default())], + hashed_account_keys: vec![], + hashed_storage_keys: vec![HashedStorageKey::new(B256::ZERO, B256::ZERO)], + }; + + // Insert entries + cursor.insert(block_1, &entry1).unwrap(); + cursor.insert(block_2, &entry2).unwrap(); + + // Read entries + let mut walker = cursor.walk(Some(block_1)).unwrap(); + let mut entries = vec![walker.next().unwrap().unwrap().1]; + if let Some(Ok((_, val))) = walker.next() { + entries.push(val); + } + entries.sort(); + let mut expected = vec![entry1.clone(), entry2.clone()]; + expected.sort(); + assert_eq!(entries, expected); + + // Delete entry1 + let mut walker = cursor.walk(Some(block_1)).unwrap(); + while let Some(Ok((_, val))) = walker.next() { + if val == entry1 { + walker.delete_current().unwrap(); + break; + } + } + + // Verify delete + let mut walker = cursor.walk(Some(block_1)).unwrap(); + assert_eq!(walker.next().unwrap().unwrap().1, entry2); + assert!(walker.next().is_none()); + } + + #[test] + fn store_trie_updates_deleted_account_trie() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(7, B256::ZERO)); + + // Prepare a BlockStateDiff that removes an account trie node at `acc_path` + let acc_path = Nibbles::from_nibbles_unchecked([0x0A, 0x0B, 0x0C]); + let mut diff_trie_updates = TrieUpdates::default(); + diff_trie_updates.removed_nodes.insert(acc_path); + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify deletion was written at BLOCK + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let vv = cur + .seek_by_key_subkey(StoredNibbles::from(acc_path), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!(vv.value.0.is_none(), "expected account trie deletion"); + } + + #[test] + fn store_trie_updates_deleted_storage_trie() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(8, B256::ZERO)); + + // Prepare a BlockStateDiff that removes a storage trie node for `addr` at `st_path` + let addr = B256::from([0xAB; 32]); + let st_path = Nibbles::from_nibbles_unchecked([0x01, 0x02, 0x03]); + + let mut diff_trie_updates = TrieUpdates::default(); + let mut st_updates = reth_trie::updates::StorageTrieUpdates::default(); + // mark this storage trie node as removed + st_updates.removed_nodes.insert(st_path); + diff_trie_updates.storage_tries.insert(addr, st_updates); + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify deletion was written at BLOCK + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let key = StorageTrieKey::new(addr, StoredNibbles::from(st_path)); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!(vv.value.0.is_none(), "expected storage trie deletion"); + } + + #[test] + fn store_trie_updates_wiped_storage_trie_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr_wiped = B256::from([0x10; 32]); + let addr_live = B256::from([0xF0; 32]); + + // Seed some storage-trie nodes at block 0 for the address that will be wiped. + let p1 = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let p2 = Nibbles::from_nibbles_unchecked([0x0A, 0x0B, 0x0C]); + let n1 = BranchNodeCompact::default(); + let n2 = BranchNodeCompact::default(); + + store + .store_storage_branches(addr_wiped, vec![(p1, Some(n1)), (p2, Some(n2))]) + .expect("seed wiped addr trie nodes"); + + // Build a BlockStateDiff that wipes addr_wiped's storage trie, and + // also adds a normal storage-trie node for addr_live. + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(123, B256::ZERO)); + let mut diff_trie_updates = TrieUpdates::default(); + + // Wipe for addr_wiped + let mut wiped_updates = StorageTrieUpdates::default(); + wiped_updates.set_deleted(true); + diff_trie_updates + .storage_tries + .insert(addr_wiped, wiped_updates); + + // Normal update for addr_live + let live_path = Nibbles::from_nibbles_unchecked([0xEE, 0xFF]); + let live_node = BranchNodeCompact::default(); + let mut live_updates = StorageTrieUpdates::default(); + live_updates.storage_nodes.insert(live_path, live_node); + diff_trie_updates + .storage_tries + .insert(addr_live, live_updates); + + // Execute the store + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify: for addr_wiped, each previously existing path now has a deletion tombstone at + // BLOCK. + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for path in [p1, p2] { + let key = StorageTrieKey::new(addr_wiped, StoredNibbles::from(path)); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_none(), + "expected tombstone at wipe block for path {path:?}" + ); + } + } + + // Verify: addr_live got its normal node written at BLOCK (not a deletion). + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let key = StorageTrieKey::new(addr_live, StoredNibbles::from(live_path)); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_some(), + "expected normal node for non-wiped address at BLOCK" + ); + } + } + + /// Mirror of `tests/lib.rs::test_store_trie_updates_with_wiped_storage_and_new_slots` for + /// the storage *trie* path. When `StorageTrieUpdates::is_deleted` is true AND + /// `storage_nodes` is non-empty (the shape revm/reth produce when a contract is destroyed + /// and recreated with a fresh trie in the same block), the wipe branch must tombstone every + /// pre-existing path for the address AND persist the new post-recreation nodes — with new + /// nodes winning on path collision. + #[test] + fn store_trie_updates_wiped_storage_trie_with_new_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x77; 32]); + let pre_path_dropped = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let pre_path_overwritten = Nibbles::from_nibbles_unchecked([0x0A, 0x0B]); + let new_path_kept = Nibbles::from_nibbles_unchecked([0xCC, 0xDD]); + + store + .store_storage_branches( + addr, + vec![ + (pre_path_dropped, Some(BranchNodeCompact::default())), + (pre_path_overwritten, Some(BranchNodeCompact::default())), + ], + ) + .expect("seed"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(123, B256::ZERO)); + let mut diff_trie_updates = TrieUpdates::default(); + let mut wiped_with_new = StorageTrieUpdates::default(); + wiped_with_new.set_deleted(true); + wiped_with_new + .storage_nodes + .insert(new_path_kept, BranchNodeCompact::default()); + wiped_with_new + .storage_nodes + .insert(pre_path_overwritten, BranchNodeCompact::default()); + diff_trie_updates.storage_tries.insert(addr, wiped_with_new); + + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let dropped_key = StorageTrieKey::new(addr, StoredNibbles::from(pre_path_dropped)); + let dropped_vv = cur + .seek_by_key_subkey(dropped_key, BLOCK.block.number) + .expect("seek") + .expect("tombstone exists"); + assert_eq!(dropped_vv.block_number, BLOCK.block.number); + assert!( + dropped_vv.value.0.is_none(), + "pre-existing path with no new node must tombstone" + ); + + let overwritten_key = StorageTrieKey::new(addr, StoredNibbles::from(pre_path_overwritten)); + let overwritten_vv = cur + .seek_by_key_subkey(overwritten_key, BLOCK.block.number) + .expect("seek") + .expect("overlay exists"); + assert_eq!(overwritten_vv.block_number, BLOCK.block.number); + assert!( + overwritten_vv.value.0.is_some(), + "collision path must reflect the new node, not the wipe tombstone", + ); + + let new_key = StorageTrieKey::new(addr, StoredNibbles::from(new_path_kept)); + let new_vv = cur + .seek_by_key_subkey(new_key, BLOCK.block.number) + .expect("seek") + .expect("new node exists"); + assert_eq!(new_vv.block_number, BLOCK.block.number); + assert!( + new_vv.value.0.is_some(), + "new path must be persisted, not dropped by the wipe" + ); + } + + #[test] + fn store_trie_updates_wiped_storage() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // We'll pre-seed storage at block 0, then issue a wipe at BLOCK. + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(42, B256::ZERO)); + + let addr = B256::from([0x55; 32]); + let s1 = B256::from([0x01; 32]); + let s2 = B256::from([0x02; 32]); + let v1 = U256::from(111u64); + let v2 = U256::from(222u64); + + // Seed prior storage (block_number = 0 in store_hashed_storages) + store + .store_hashed_storages(addr, vec![(s1, v1), (s2, v2)]) + .expect("seed"); + + // Build BlockStateDiff that marks this address as wiped at BLOCK + let mut diff_post_state = HashedPostState::default(); + + let wiped = reth_trie::HashedStorage::new(true); + + diff_post_state.storages.insert(addr, wiped); + + // Execute + let diff = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_post_state.into_sorted(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify: for each pre-existing slot, there should be a tombstone (MaybeDeleted(None)) at + // BLOCK. + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for slot in [s1, s2] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_none(), + "expected deletion tombstone for slot {:?} at block {}", + slot, + BLOCK.block.number, + ); + } + } + + #[test] + fn store_trie_updates_wiped_and_non_wiped_mixed_order() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Choose addresses so that wiped < non_wiped in sort order (your impl sorts by address) + let addr_wiped = B256::from([0x01; 32]); // will sort first + let addr_live = B256::from([0xF0; 32]); // will sort later + + // Slots & values + let ws1 = B256::from([0xA1; 32]); + let ws2 = B256::from([0xA2; 32]); + let wv1 = U256::from(111u64); + let wv2 = U256::from(222u64); + + let ls1 = B256::from([0xB1; 32]); + let lv1_old = U256::from(333u64); + let lv1_new = U256::from(999u64); // will be written at BLOCK + + // Seed prior storage at block 0 for BOTH addresses + store + .store_hashed_storages(addr_wiped, vec![(ws1, wv1), (ws2, wv2)]) + .expect("seed wiped addr"); + store + .store_hashed_storages(addr_live, vec![(ls1, lv1_old)]) + .expect("seed live addr"); + + // Build diff: wiped first (by address sort), then non-wiped with a write + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(77, B256::ZERO)); + let mut diff_post_state = HashedPostState::default(); + + // Wiped storage for addr_wiped + let wiped = reth_trie::HashedStorage::new(true); + diff_post_state.storages.insert(addr_wiped, wiped); + + // Non-wiped storage for addr_live (append new value) + let mut live = reth_trie::HashedStorage::default(); + live.storage.insert(ls1, lv1_new); + diff_post_state.storages.insert(addr_live, live); + + // Execute + let diff = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_post_state.into_sorted(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify: wiped address got tombstones at BLOCK for each pre-existing slot + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + for slot in [ws1, ws2] { + let key = HashedStorageKey::new(addr_wiped, slot); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_none(), + "expected deletion tombstone for wiped slot {:?} at block {}", + slot, + BLOCK.block.number, + ); + } + } + + // Verify: non-wiped address got the new value at BLOCK (not a deletion) + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let key = HashedStorageKey::new(addr_live, ls1); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!( + inner.0, lv1_new, + "expected updated value for non-wiped address" + ); + } + } + + #[test] + fn test_proof_window() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Test initial state (no values set) + let initial_value = store.get_earliest_block_number().expect("get earliest"); + assert_eq!(initial_value, None); + + // Test setting the value + let block_number = 42u64; + let hash = B256::random(); + store + .set_earliest_block_number(block_number, hash) + .expect("set earliest"); + + // Verify value was stored correctly + let retrieved = store.get_earliest_block_number().expect("get earliest"); + assert_eq!(retrieved, Some((block_number, hash))); + + // Test updating with new values + let new_block_number = 100u64; + let new_hash = B256::random(); + store + .set_earliest_block_number(new_block_number, new_hash) + .expect("update earliest"); + + // Verify update worked + let updated = store + .get_earliest_block_number() + .expect("get updated earliest"); + assert_eq!(updated, Some((new_block_number, new_hash))); + + // Verify that latest_block falls back to earliest when not set + let latest = store.get_latest_block_number().expect("get latest"); + assert_eq!( + latest, + Some((new_block_number, new_hash)), + "Latest block should fall back to earliest when not explicitly set" + ); + } + + #[test] + fn replace_updates_prunes_and_adds_new_chain() { + let dir = tempfile::TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Test address and helper to make diffs with distinct nonces. + let addr = B256::from([0xAB; 32]); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // --- Build initial canonical chain: 1 -> 2 -> 3 --- + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Sanity: entries for 1,2,3 exist with expected nonces. + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let v1 = cur + .seek_by_key_subkey(addr, 1) + .expect("seek") + .expect("exists"); + let v2 = cur + .seek_by_key_subkey(addr, 2) + .expect("seek") + .expect("exists"); + let v3 = cur + .seek_by_key_subkey(addr, 3) + .expect("seek") + .expect("exists"); + assert_eq!(v1.value.0.unwrap().nonce, 10); + assert_eq!(v2.value.0.unwrap().nonce, 20); + assert_eq!(v3.value.0.unwrap().nonce, 30); + } + + // --- Reorg at LCA = 2: prune >2, then add 3' and 4' --- + let b3p = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); // 3' + let b4p = BlockWithParent::new(b3p.block.hash, NumHash::new(4, B256::random())); // 4' + + // Build blocks_to_add vec. + let blocks_to_add = vec![(b3p, make_diff(300)), (b4p, make_diff(400))]; + + store + .replace_updates(BlockNumHash::new(2, b2.block.hash), blocks_to_add) + .expect("replace_updates succeeds"); + + // --- Verify post-conditions --- + + // 1) HashedAccountHistory: blocks 1,2 remain; block 3 was replaced (nonce=300); block 4 + // exists (nonce=400). + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur + .seek_by_key_subkey(addr, 1) + .expect("seek") + .expect("exists"); + assert_eq!(v1.value.0.unwrap().nonce, 10); + + let v2 = cur + .seek_by_key_subkey(addr, 2) + .expect("seek") + .expect("exists"); + assert_eq!(v2.value.0.unwrap().nonce, 20); + + // Old block 3 (nonce=30) should have been pruned; we should now have a fresh entry at 3 + // with nonce=300. + let v3_new = cur + .seek_by_key_subkey(addr, 3) + .expect("seek") + .expect("replaced exists"); + assert_eq!( + v3_new.value.0.unwrap().nonce, + 300, + "block 3 should be replaced by new chain" + ); + + // New block 4 should exist. + let v4 = cur + .seek_by_key_subkey(addr, 4) + .expect("seek") + .expect("exists"); + assert_eq!(v4.value.0.unwrap().nonce, 400); + } + + // 2) BlockChangeSet contains exactly entries for block numbers {1,2,3,4}. + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let mut seen = std::collections::BTreeSet::new(); + let mut it = cur.walk(Some(1)).expect("walk"); + while let Some(Ok((bn, _))) = it.next() { + seen.insert(bn); + } + let expected: std::collections::BTreeSet = [1u64, 2, 3, 4].into_iter().collect(); + assert_eq!( + seen, expected, + "BlockChangeSet should reflect pruned+new chain" + ); + } + } + + #[test] + fn test_unwind_history_basic() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 -> 4 + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let b4 = BlockWithParent::new(b3.block.hash, NumHash::new(4, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + store + .store_trie_updates(b4, make_diff(40)) + .expect("store b4"); + + // Unwind to block 2 + store.unwind_history(b2).expect("unwind"); + + // Verify: blocks 1 and 2 remain, blocks 3 and 4 are removed + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + assert!( + cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 2).unwrap().is_none(), + "Block 2 should be removed" + ); + assert!( + cur.seek_by_key_subkey(addr, 3).unwrap().is_none(), + "Block 3 should be removed" + ); + assert!( + cur.seek_by_key_subkey(addr, 4).unwrap().is_none(), + "Block 4 should be removed" + ); + + // Verify ProofWindow LatestBlock is updated + let mut proof_window_cur = tx.cursor_read::().expect("cursor"); + let latest = proof_window_cur + .seek_exact(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("latest exists"); + assert_eq!(latest.1.number(), 1); + assert_eq!(*latest.1.hash(), b2.parent); + } + + #[test] + fn test_unwind_history_to_earliest() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b1.block.number, b1.block.hash) + .expect("set earliest"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Unwind to block b1 + let res = store.unwind_history(b1); + // should fail as we cannot unwind past earliest block + assert!(res.is_err(), "unwind to earliest block should error"); + assert!(matches!( + res.unwrap_err(), + MorphProofsStorageError::UnwindBeyondEarliest { .. } + )); + } + + #[test] + fn test_unwind_history_with_storage() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let slot = B256::random(); + + let make_diff = |nonce: u64, slot_value: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + let mut storage = HashedStorage::default(); + storage.storage.insert(slot, U256::from(slot_value)); + d_post_state.storages.insert(addr, storage); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain with storage changes + let b0 = BlockWithParent::new(B256::ZERO, NumHash::new(0, B256::random())); + let b1 = BlockWithParent::new(b0.block.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b0.block.number, b0.block.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10, 100)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20, 200)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30, 300)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("unwind"); + + // Verify account history + let tx = store.env.tx().expect("tx"); + let mut acc_cur = tx.new_cursor::().expect("cursor"); + assert!( + acc_cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + acc_cur.seek_by_key_subkey(addr, 2).unwrap().is_none(), + "Block 2 should be removed" + ); + assert!( + acc_cur.seek_by_key_subkey(addr, 3).unwrap().is_none(), + "Block 3 should be removed" + ); + + // Verify storage history + let mut storage_cur = tx.new_cursor::().expect("cursor"); + let storage_key = HashedStorageKey::new(addr, slot); + assert!( + storage_cur + .seek_by_key_subkey(storage_key.clone(), 1) + .unwrap() + .is_some(), + "Storage at block 1 should remain" + ); + assert!( + storage_cur + .seek_by_key_subkey(storage_key.clone(), 2) + .unwrap() + .is_none(), + "Storage at block 2 should be removed" + ); + assert!( + storage_cur + .seek_by_key_subkey(storage_key, 3) + .unwrap() + .is_none(), + "Storage at block 3 should be removed" + ); + } + + #[test] + fn test_unwind_history_with_trie_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let path1 = Nibbles::from_nibbles_unchecked([0x01]); + let path2 = Nibbles::from_nibbles_unchecked([0x02]); + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let node2 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + + let make_diff = |path: Nibbles, node: BranchNodeCompact| { + let mut d_trie_updates = TrieUpdates::default(); + d_trie_updates.account_nodes.insert(path, node); + BlockStateDiff { + sorted_trie_updates: d_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + } + }; + + // Build chain with trie updates + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(path1, node1)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(path2, node2.clone())) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(path1, node2)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("unwind"); + + // Verify trie node history + let tx = store.env.tx().expect("tx"); + let mut trie_cur = tx.cursor_dup_read::().expect("cursor"); + + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 1) + .unwrap() + .is_some(), + "Trie node at block 1 should remain" + ); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path2), 2) + .unwrap() + .is_none(), + "Trie node at block 2 should be removed" + ); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 3) + .unwrap() + .is_none(), + "Trie node at block 3 should be removed" + ); + } + + #[test] + fn test_unwind_history_comprehensive() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Setup comprehensive scenario with accounts, storage, and trie nodes + let addr1 = B256::random(); + let addr2 = B256::random(); + let slot1 = B256::random(); + let slot2 = B256::random(); + let path1 = Nibbles::from_nibbles_unchecked([0x01]); + let path2 = Nibbles::from_nibbles_unchecked([0x02]); + let storage_path1 = Nibbles::from_nibbles_unchecked([0x03]); + + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let node2 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + let storage_node1 = BranchNodeCompact::new(0b100, 0, 0, vec![], Some(B256::random())); + + // Block 0: Set earliest block + let b0 = NumHash::new(0, B256::random()); + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + + // Block 1: Insert multiple types of data + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let mut diff1_trie_updates = TrieUpdates::default(); + let mut diff1_post_state = HashedPostState::default(); + diff1_post_state.accounts.insert(addr1, Some(acc1)); + diff1_trie_updates.account_nodes.insert(path1, node1); + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, U256::from(1111)); + diff1_post_state.storages.insert(addr1, storage1); + let mut storage_updates1 = StorageTrieUpdates::default(); + storage_updates1 + .storage_nodes + .insert(storage_path1, storage_node1); + diff1_trie_updates + .storage_tries + .insert(addr1, storage_updates1); + let diff1 = BlockStateDiff { + sorted_trie_updates: diff1_trie_updates.into_sorted(), + sorted_post_state: diff1_post_state.into_sorted(), + }; + store.store_trie_updates(b1, diff1).expect("store b1"); + + // Block 2: More updates + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let mut diff2_trie_updates = TrieUpdates::default(); + let mut diff2_post_state = HashedPostState::default(); + diff2_post_state.accounts.insert(addr2, Some(acc2)); + diff2_trie_updates.account_nodes.insert(path2, node2); + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot2, U256::from(2222)); + diff2_post_state.storages.insert(addr2, storage2); + let diff2 = BlockStateDiff { + sorted_trie_updates: diff2_trie_updates.into_sorted(), + sorted_post_state: diff2_post_state.into_sorted(), + }; + store.store_trie_updates(b2, diff2).expect("store b2"); + + // Block 3: Additional updates + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let mut diff3_post_state = HashedPostState::default(); + diff3_post_state.accounts.insert(addr1, Some(acc2)); // update addr1 + let diff3 = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff3_post_state.into_sorted(), + }; + store.store_trie_updates(b3, diff3).expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("unwind"); + + let tx = store.env.tx().expect("tx"); + + // Verify account history: block 1 remains, blocks 2 and 3 removed + let mut acc_cur = tx.new_cursor::().expect("cursor"); + assert!(acc_cur.seek_by_key_subkey(addr1, 1).unwrap().is_some()); + assert!(acc_cur.seek_by_key_subkey(addr2, 2).unwrap().is_none()); + assert!(acc_cur.seek_by_key_subkey(addr1, 3).unwrap().is_none()); + + // Verify trie history + let mut trie_cur = tx.cursor_dup_read::().expect("cursor"); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 1) + .unwrap() + .is_some() + ); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path2), 2) + .unwrap() + .is_none() + ); + + // Verify storage history + let mut storage_cur = tx.new_cursor::().expect("cursor"); + assert!( + storage_cur + .seek_by_key_subkey(HashedStorageKey::new(addr1, slot1), 1) + .unwrap() + .is_some() + ); + assert!( + storage_cur + .seek_by_key_subkey(HashedStorageKey::new(addr2, slot2), 2) + .unwrap() + .is_none() + ); + + // Verify storage trie history + let mut storage_trie_cur = tx.cursor_dup_read::().expect("cursor"); + assert!( + storage_trie_cur + .seek_by_key_subkey( + StorageTrieKey::new(addr1, StoredNibbles::from(storage_path1)), + 1 + ) + .unwrap() + .is_some() + ); + + // Verify ProofWindow LatestBlock is updated + let mut proof_window_cur = tx.cursor_read::().expect("cursor"); + let latest = proof_window_cur + .seek_exact(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("latest exists"); + assert_eq!(latest.1.number(), 1); + assert_eq!(*latest.1.hash(), b1.block.hash); + + // Verify change sets are removed for blocks > 1 + let mut change_cur = tx.new_cursor::().expect("cursor"); + assert!( + change_cur.seek_exact(1).unwrap().is_some(), + "Block 1 changeset should remain" + ); + assert!( + change_cur.seek_exact(2).unwrap().is_none(), + "Block 2 changeset should be removed" + ); + assert!( + change_cur.seek_exact(3).unwrap().is_none(), + "Block 3 changeset should be removed" + ); + } + + #[test] + fn test_unwind_history_empty_chain() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Try to unwind when there's nothing stored yet + let unwind_to = BlockWithParent::new(B256::ZERO, NumHash::new(0, B256::ZERO)); + let result = store.unwind_history(unwind_to); + + // Should succeed (no-op) + assert!(result.is_ok(), "Unwinding empty chain should succeed"); + } + + #[test] + fn test_unwind_history_idempotent() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("first unwind"); + + // Unwind again to the same block (should be idempotent) + store.unwind_history(b2).expect("second unwind"); + + // Verify state is still correct + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + assert!( + cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 2).unwrap().is_none(), + "Block 2 should be removed" + ); + assert!( + cur.seek_by_key_subkey(addr, 3).unwrap().is_none(), + "Block 3 should be removed" + ); + } + + #[test] + fn test_unwind_history_beyond_latest() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let b4 = BlockWithParent::new(b3.block.hash, NumHash::new(4, B256::random())); + let b5 = BlockWithParent::new(b4.block.hash, NumHash::new(5, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b5).expect("first unwind"); + + // Verify state is still correct + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + assert!( + cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 2).unwrap().is_some(), + "Block 2 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 3).unwrap().is_some(), + "Block 3 should remain" + ); + + let mut proof_window_cur = tx.cursor_read::().expect("cursor"); + let latest = proof_window_cur + .seek_exact(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("latest exists"); + assert_eq!(latest.1.number(), b3.block.number); + assert_eq!(*latest.1.hash(), b3.block.hash); + } +} diff --git a/crates/proofs/src/error.rs b/crates/proofs/src/error.rs new file mode 100644 index 0000000..03a3aee --- /dev/null +++ b/crates/proofs/src/error.rs @@ -0,0 +1,233 @@ +//! Errors interfacing with [`MorphProofsStore`](crate::MorphProofsStore) type. + +use std::sync::Arc; + +use alloy_primitives::B256; +use reth_codecs::DecompressError; +use reth_db::DatabaseError; +use reth_execution_errors::BlockExecutionError; +use reth_provider::ProviderError; +use reth_trie_common::Nibbles; +use thiserror::Error; + +/// Error type for storage operations +#[derive(Debug, Clone, Error)] +pub enum MorphProofsStorageError { + /// The proof DB belongs to another canonical chain. + #[error("proof database chain identity mismatch: {0}")] + ChainIdentityMismatch(&'static str), + /// The proof DB schema cannot be read by this binary. + #[error("proof database schema mismatch: expected {expected}, found {actual}")] + SchemaMismatch { + /// Schema expected by this binary. + expected: u32, + /// Schema persisted in the database. + actual: u32, + }, + /// Durable metadata is missing or malformed. + #[error("corrupt proof database metadata: {0}")] + CorruptMetadata(&'static str), + /// Only part of the durable identity metadata is present. + #[error( + "proof database identity metadata is incomplete; manually delete the proof database before reinitializing" + )] + IncompleteIdentityMetadata, + /// Proof/history data exists without the metadata needed to validate its identity. + #[error( + "proof database contains proof/history data but identity metadata is missing; manually delete the proof database before reinitializing" + )] + ProofDataWithoutIdentityMetadata, + /// No blocks found + #[error("No blocks found")] + NoBlocksFound, + /// Parent block number is less than earliest stored block number + #[error("Parent block number is less than earliest stored block number")] + UnknownParent, + /// Block is out of order + #[error( + "Block {block_number} is out of order (parent: {parent_block_hash}, latest stored block hash: {latest_block_hash})" + )] + OutOfOrder { + /// The block number being inserted + block_number: u64, + /// The parent hash of the block being inserted + parent_block_hash: B256, + /// block hash of the latest stored block + latest_block_hash: B256, + }, + /// Block update failed since parent state + #[error( + "Cannot execute block updates for block {block_number} without parent state {parent_block_number} (latest stored block number: {latest_block_number})" + )] + MissingParentBlock { + /// The block number being executed + block_number: u64, + /// The parent state of the block being executed + parent_block_number: u64, + /// Latest stored block number + latest_block_number: u64, + }, + /// State root mismatch + #[error( + "State root mismatch for block {block_number} (have: {current_state_hash}, expected: {expected_state_hash})" + )] + StateRootMismatch { + /// Block number + block_number: u64, + /// Have state root + current_state_hash: B256, + /// Expected state root + expected_state_hash: B256, + }, + /// No change set for block + #[error("No change set found for block {0}")] + NoChangeSetForBlock(u64), + /// Missing account trie history for a specific path at a specific block number + #[error("Missing account trie history for path {0:?} at block {1}")] + MissingAccountTrieHistory(Nibbles, u64), + /// Missing storage trie history for a specific address and path at a specific block number + #[error("Missing storage trie history for address {0:?}, path {1:?} at block {2}")] + MissingStorageTrieHistory(B256, Nibbles, u64), + /// Missing hashed account history for a specific key at a specific block number + #[error("Missing hashed account history for key {0:?} at block {1}")] + MissingHashedAccountHistory(B256, u64), + /// Missing hashed storage history for a specific address and key at a specific block number + #[error( + "Missing hashed storage history for address {hashed_address:?}, key {hashed_storage_key:?} at block {block_number}" + )] + MissingHashedStorageHistory { + /// The hashed address + hashed_address: B256, + /// The hashed storage key + hashed_storage_key: B256, + /// The block number + block_number: u64, + }, + /// Attempted to unwind to a block beyond the earliest stored block + #[error( + "Attempted to unwind to block {unwind_block_number} beyond earliest stored block {earliest_block_number}" + )] + UnwindBeyondEarliest { + /// The block number being unwound to + unwind_block_number: u64, + /// The earliest stored block number + earliest_block_number: u64, + }, + /// Error occurred while interacting with the database. + #[error(transparent)] + DatabaseError(DatabaseError), + /// Error occurred while trying to acquire a lock. + #[error("failed lock attempt")] + TryLockError, + /// Error occurred during block execution. + #[error(transparent)] + ExecutionError(Arc), + /// Error occurred while interacting with the provider. + #[error(transparent)] + ProviderError(Arc), + /// Initialization detected inconsistent state between proofs storage and source DB. + #[error( + "Initialization Proofs storage detected inconsistent state. Storage does not match source DB. \ + Please clear proofs data and retry initialization." + )] + InitializeStorageInconsistentState, + /// Batch session used after its underlying transaction has been committed or aborted. + #[error("Batch session used after its underlying transaction was closed")] + BatchSessionClosed, +} + +impl From for MorphProofsStorageError { + fn from(error: BlockExecutionError) -> Self { + Self::ExecutionError(Arc::new(error)) + } +} + +impl From for MorphProofsStorageError { + fn from(error: ProviderError) -> Self { + Self::ProviderError(Arc::new(error)) + } +} + +impl From for DatabaseError { + fn from(error: MorphProofsStorageError) -> Self { + match error { + MorphProofsStorageError::DatabaseError(err) => err, + _ => Self::Custom(Arc::new(error)), + } + } +} + +impl From for MorphProofsStorageError { + fn from(error: DatabaseError) -> Self { + if let DatabaseError::Custom(ref err) = error + && let Some(err) = err.downcast_ref::() + { + return err.clone(); + } + Self::DatabaseError(error) + } +} + +impl From for MorphProofsStorageError { + fn from(error: DecompressError) -> Self { + Self::DatabaseError(DatabaseError::Other(format!("decompress error: {error}"))) + } +} + +/// Result type for storage operations +pub type MorphProofsStorageResult = Result; + +#[cfg(test)] +mod tests { + use reth_execution_errors::BlockValidationError; + + use super::*; + + #[test] + fn test_morph_proofs_store_error_to_db_error() { + let original_error = MorphProofsStorageError::NoBlocksFound; + + let db_error = DatabaseError::from(original_error); + assert!(matches!(db_error, DatabaseError::Custom(_))); + + let converted_error = MorphProofsStorageError::from(db_error); + assert!(matches!( + converted_error, + MorphProofsStorageError::NoBlocksFound + )) + } + + #[test] + fn test_db_error_to_morph_proofs_store_error() { + let original_error = DatabaseError::Decode; + + let morph_proofs_store_error = MorphProofsStorageError::from(original_error); + assert!(matches!( + morph_proofs_store_error, + MorphProofsStorageError::DatabaseError(DatabaseError::Decode) + )); + + let converted_error = DatabaseError::from(morph_proofs_store_error); + assert!(matches!(converted_error, DatabaseError::Decode)) + } + + #[test] + fn test_conversion_from_block_execution_error() { + let block_execution_error = + BlockExecutionError::Validation(BlockValidationError::IncrementBalanceFailed); + + let morph_proofs_store_error = MorphProofsStorageError::from(block_execution_error); + assert!( + matches!(morph_proofs_store_error, MorphProofsStorageError::ExecutionError(err) if matches!(*err, BlockExecutionError::Validation(BlockValidationError::IncrementBalanceFailed))) + ) + } + + #[test] + fn test_conversion_from_provider_error() { + let provider_error = ProviderError::SenderRecoveryError; + let morph_proofs_store_error = MorphProofsStorageError::from(provider_error); + assert!( + matches!(morph_proofs_store_error, MorphProofsStorageError::ProviderError(err) if matches!(*err, ProviderError::SenderRecoveryError)) + ) + } +} diff --git a/crates/proofs/src/in_memory.rs b/crates/proofs/src/in_memory.rs new file mode 100644 index 0000000..08ac678 --- /dev/null +++ b/crates/proofs/src/in_memory.rs @@ -0,0 +1,1138 @@ +//! In-memory implementation of [`MorphProofsStore`] for testing purposes + +use std::{collections::BTreeMap, sync::Arc}; + +use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; +use alloy_primitives::{B256, U256}; +use parking_lot::RwLock; +use reth_db::DatabaseError; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{ + BranchNodeCompact, HashedPostStateSorted, Nibbles, StoredNibbles, updates::TrieUpdatesSorted, +}; + +use crate::{ + BlockStateDiff, MorphProofsStorageError, MorphProofsStorageResult, MorphProofsStore, + api::{ + InitialStateAnchor, InitialStateStatus, MorphProofsBatchSession, MorphProofsBatchStore, + MorphProofsInitialStateStore, WriteCounts, + }, + db::{HashedStorageKey, StorageTrieKey}, +}; + +/// In-memory implementation of [`MorphProofsStore`] for testing purposes +#[derive(Debug, Clone)] +pub struct InMemoryProofsStorage { + /// Shared state across all instances + inner: Arc>, +} + +#[derive(Debug, Default)] +struct InMemoryStorageInner { + /// Account trie branches: (`block_number`, path) -> `branch_node` + account_branches: BTreeMap<(u64, Nibbles), Option>, + + /// Storage trie branches: (`block_number`, `hashed_address`, path) -> `branch_node` + storage_branches: BTreeMap<(u64, B256, Nibbles), Option>, + + /// Hashed accounts: (`block_number`, `hashed_address`) -> account + hashed_accounts: BTreeMap<(u64, B256), Option>, + + /// Hashed storages: (`block_number`, `hashed_address`, `hashed_slot`) -> value + hashed_storages: BTreeMap<(u64, B256, B256), U256>, + + /// Trie updates by block number + trie_updates: BTreeMap, + + /// Post state by block number + post_states: BTreeMap, + + /// Earliest block number and hash + earliest_block: Option<(u64, B256)>, + + /// The anchor block (initial state) of the store. + anchor_block: Option<(u64, B256)>, +} + +impl InMemoryStorageInner { + fn store_trie_updates( + &mut self, + block_number: u64, + block_state_diff: BlockStateDiff, + ) -> WriteCounts { + let mut result = WriteCounts::default(); + + // Store account branch nodes + for (path, branch) in block_state_diff.sorted_trie_updates.account_nodes_ref() { + self.account_branches + .insert((block_number, *path), branch.clone()); + result.account_trie_updates_written_total += 1; + } + + // Store storage branch nodes and removals + for (address, storage_trie_updates) in + block_state_diff.sorted_trie_updates.storage_tries_ref() + { + // Store storage branch nodes + for (path, branch) in storage_trie_updates.storage_nodes_ref() { + self.storage_branches + .insert((block_number, *address, *path), branch.clone()); + result.storage_trie_updates_written_total += 1; + } + } + + for (address, account) in &block_state_diff.sorted_post_state.accounts { + self.hashed_accounts + .insert((block_number, *address), *account); + result.hashed_accounts_written_total += 1; + } + + for (hashed_address, storage) in &block_state_diff.sorted_post_state.storages { + // Handle wiped storage: iterate all existing values and mark them as deleted + // This is an expensive operation and should never happen for blocks going forward. + if storage.wiped { + // Collect latest values for each slot up to the current block + let mut slot_to_latest: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for ((block, address, slot), value) in &self.hashed_storages { + if *block < block_number && *address == *hashed_address { + if let Some((existing_block, _)) = slot_to_latest.get(slot) { + if *block > *existing_block { + slot_to_latest.insert(*slot, (*block, *value)); + } + } else { + slot_to_latest.insert(*slot, (*block, *value)); + } + } + } + + // Store zero values for all non-zero slots to mark them as deleted + for (slot, (_, value)) in slot_to_latest { + if !value.is_zero() { + self.hashed_storages + .insert((block_number, *hashed_address, slot), U256::ZERO); + result.hashed_storages_written_total += 1; + } + } + } + // Persist any new slots that arrive in the same block as the wipe (e.g. SELFDESTRUCT + // + same-block re-CREATE2 produces `wiped = true` together with fresh + // `storage_slots_ref()` for the recreated contract). Mirrors the MDBX backend. + for (slot, value) in storage.storage_slots_ref() { + self.hashed_storages + .insert((block_number, *hashed_address, *slot), *value); + result.hashed_storages_written_total += 1; + } + } + + self.trie_updates + .insert(block_number, block_state_diff.sorted_trie_updates.clone()); + self.post_states + .insert(block_number, block_state_diff.sorted_post_state); + + result + } +} + +impl Default for InMemoryProofsStorage { + fn default() -> Self { + Self::new() + } +} + +impl InMemoryProofsStorage { + /// Create a new in-memory Morph proofs storage instance. + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(InMemoryStorageInner::default())), + } + } + + fn get_latest_account_trie_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .account_branches + .range(( + std::ops::Bound::Included((0, Nibbles::default())), + std::ops::Bound::Excluded((1, Nibbles::default())), + )) + .next_back() + .map(|((_, nibbles), _)| StoredNibbles::from(*nibbles))) + } + + fn get_latest_storage_trie_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .storage_branches + .range(( + std::ops::Bound::Included((0, B256::ZERO, Nibbles::default())), + std::ops::Bound::Excluded((1, B256::ZERO, Nibbles::default())), + )) + .next_back() + .map(|((_, address, nibbles), _)| { + StorageTrieKey::new(*address, StoredNibbles::from(*nibbles)) + })) + } + + fn get_latest_hashed_account_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .hashed_accounts + .range(( + std::ops::Bound::Included((0, B256::ZERO)), + std::ops::Bound::Excluded((1, B256::ZERO)), + )) + .next_back() + .map(|((_, address), _)| *address)) + } + + fn get_latest_hashed_storage_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .hashed_storages + .range(( + std::ops::Bound::Included((0, B256::ZERO, B256::ZERO)), + std::ops::Bound::Excluded((1, B256::ZERO, B256::ZERO)), + )) + .next_back() + .map(|((_, address, slot), _)| HashedStorageKey::new(*address, *slot))) + } +} + +/// In-memory implementation of [`TrieCursor`]. +#[derive(Debug)] +pub struct InMemoryTrieCursor { + /// inner storage reference + inner: Arc>, + /// Hashed address being queried + hashed_address: Option, + /// max block number for the cursor + max_block_number: u64, + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + + /// Whether the entries have been populated + is_populated: bool, + /// Sorted entries that match the query parameters + entries: Vec<(Nibbles, BranchNodeCompact)>, +} + +impl InMemoryTrieCursor { + const fn new( + inner: Arc>, + hashed_address: Option, + max_block_number: u64, + ) -> Self { + Self { + inner, + hashed_address, + max_block_number, + position: -1, + + is_populated: false, + entries: Vec::new(), + } + } + + fn ensure_entries_populated(&mut self) -> Result<(), DatabaseError> { + if self.is_populated { + return Ok(()); + } + + let storage = self + .inner + .try_read() + .ok_or(MorphProofsStorageError::TryLockError)?; + + // Common logic: collect latest values for each path + let mut path_to_latest: std::collections::BTreeMap< + Nibbles, + (u64, Option), + > = std::collections::BTreeMap::new(); + + let mut collected_entries: Vec<(Nibbles, BranchNodeCompact)> = + if let Some(addr) = self.hashed_address { + // Storage trie cursor + for ((block, address, path), branch) in &storage.storage_branches { + if *block <= self.max_block_number && *address == addr { + if let Some((existing_block, _)) = path_to_latest.get(path) { + if *block > *existing_block { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } else { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } + } + + path_to_latest + .into_iter() + .filter_map(|(path, (_, branch))| branch.map(|b| (path, b))) + .collect() + } else { + // Account trie cursor + for ((block, path), branch) in &storage.account_branches { + if *block <= self.max_block_number { + if let Some((existing_block, _)) = path_to_latest.get(path) { + if *block > *existing_block { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } else { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } + } + + path_to_latest + .into_iter() + .filter_map(|(path, (_, branch))| branch.map(|b| (path, b))) + .collect() + }; + + // Sort by path for consistent ordering + collected_entries.sort_by_key(|(a, _)| *a); + self.entries = collected_entries; + self.is_populated = true; + Ok(()) + } +} + +impl TrieCursor for InMemoryTrieCursor { + fn seek_exact( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if let Some(pos) = self.entries.iter().position(|(p, _)| *p == path) { + self.position = pos as isize; + Ok(Some(self.entries[pos].clone())) + } else { + Ok(None) + } + } + + fn seek( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if let Some(pos) = self.entries.iter().position(|(p, _)| *p >= path) { + self.position = pos as isize; + Ok(Some(self.entries[pos].clone())) + } else { + Ok(None) + } + } + + fn next(&mut self) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize].clone())) + } else { + Ok(None) + } + } + + fn current(&mut self) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize].0)) + } else { + Ok(None) + } + } + + fn reset(&mut self) { + self.position = -1; + } +} + +impl TrieStorageCursor for InMemoryTrieCursor { + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = Some(hashed_address); + self.is_populated = false; + self.entries.clear(); + self.reset(); + } +} + +/// In-memory implementation of [`HashedCursor`] for storage slots +#[derive(Debug)] +pub struct InMemoryStorageCursor { + /// inner storage reference + inner: Arc>, + /// hashed address for which the cursor is iterating + hashed_address: B256, + /// max block number for the cursor + max_block_number: u64, + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + + /// Whether the entries have been populated + is_populated: bool, + /// Sorted entries that match the query parameters + entries: Vec<(B256, U256)>, +} + +impl InMemoryStorageCursor { + const fn new( + storage: Arc>, + hashed_address: B256, + max_block_number: u64, + ) -> Self { + Self { + inner: storage, + hashed_address, + max_block_number, + position: -1, + + is_populated: false, + entries: Vec::new(), + } + } + + fn ensure_entries_populated(&mut self) -> Result<(), DatabaseError> { + if self.is_populated { + return Ok(()); + } + + let storage = self + .inner + .try_read() + .ok_or(MorphProofsStorageError::TryLockError)?; + + // Collect latest values for each slot + let mut slot_to_latest: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for ((block, address, slot), value) in &storage.hashed_storages { + if *block <= self.max_block_number && *address == self.hashed_address { + if let Some((existing_block, _)) = slot_to_latest.get(slot) { + if *block > *existing_block { + slot_to_latest.insert(*slot, (*block, *value)); + } + } else { + slot_to_latest.insert(*slot, (*block, *value)); + } + } + } + + // Filter out zero values - they represent deleted/empty storage slots + let mut entries: Vec<(B256, U256)> = slot_to_latest + .into_iter() + .filter_map(|(slot, (_, value))| { + if value.is_zero() { + None + } else { + Some((slot, value)) + } + }) + .collect(); + + entries.sort_by_key(|(slot, _)| *slot); + self.entries = entries; + self.is_populated = true; + Ok(()) + } +} + +impl HashedCursor for InMemoryStorageCursor { + type Value = U256; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if let Some(pos) = self.entries.iter().position(|(k, _)| *k >= key) { + self.position = pos as isize; + Ok(Some(self.entries[pos])) + } else { + Ok(None) + } + } + + fn next(&mut self) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize])) + } else { + Ok(None) + } + } + + fn reset(&mut self) { + self.position = -1; + } +} + +impl HashedStorageCursor for InMemoryStorageCursor { + fn is_storage_empty(&mut self) -> Result { + Ok(self.seek(B256::ZERO)?.is_none()) + } + + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = hashed_address; + self.is_populated = false; + self.entries.clear(); + self.reset(); + } +} + +/// In-memory implementation of [`HashedCursor`] for accounts +#[derive(Debug)] +pub struct InMemoryAccountCursor { + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + /// Sorted entries that match the query parameters + entries: Vec<(B256, Account)>, +} + +impl InMemoryAccountCursor { + fn new(storage: &InMemoryStorageInner, max_block_number: u64) -> Self { + // Collect latest accounts for each address + let mut addr_to_latest: std::collections::BTreeMap)> = + std::collections::BTreeMap::new(); + + for ((block, address), account) in &storage.hashed_accounts { + if *block <= max_block_number { + if let Some((existing_block, _)) = addr_to_latest.get(address) { + if *block > *existing_block { + addr_to_latest.insert(*address, (*block, *account)); + } + } else { + addr_to_latest.insert(*address, (*block, *account)); + } + } + } + + let mut entries: Vec<(B256, Account)> = addr_to_latest + .into_iter() + .filter_map(|(address, (_, account))| account.map(|acc| (address, acc))) + .collect(); + + entries.sort_by_key(|(address, _)| *address); + + Self { + position: -1, + entries, + } + } +} + +impl HashedCursor for InMemoryAccountCursor { + type Value = Account; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + if let Some(pos) = self.entries.iter().position(|(k, _)| *k >= key) { + self.position = pos as isize; + Ok(Some(self.entries[pos])) + } else { + Ok(None) + } + } + + fn next(&mut self) -> Result, DatabaseError> { + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize])) + } else { + Ok(None) + } + } + + fn reset(&mut self) { + // no reset needed + } +} + +impl MorphProofsStore for InMemoryProofsStorage { + type StorageTrieCursor<'tx> = InMemoryTrieCursor; + type AccountTrieCursor<'tx> = InMemoryTrieCursor; + type StorageCursor<'tx> = InMemoryStorageCursor; + type AccountHashedCursor<'tx> = InMemoryAccountCursor; + type Tx<'tx> + = () + where + Self: 'tx; + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner.earliest_block) + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + // Find the latest block number from trie_updates + let latest_block = inner.trie_updates.keys().max().copied(); + Ok(latest_block.map_or(inner.earliest_block, |block| Some((block, B256::ZERO)))) + } + + fn get_earliest_block_number_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.get_earliest_block_number() + } + + fn get_latest_block_number_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.get_latest_block_number() + } + + fn storage_trie_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(InMemoryTrieCursor::new( + Arc::clone(&self.inner), + Some(hashed_address), + max_block_number, + )) + } + + fn account_trie_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(InMemoryTrieCursor::new( + Arc::clone(&self.inner), + None, + max_block_number, + )) + } + + fn storage_hashed_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(InMemoryStorageCursor::new( + Arc::clone(&self.inner), + hashed_address, + max_block_number, + )) + } + + fn account_hashed_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let inner = self + .inner + .try_read() + .ok_or(MorphProofsStorageError::TryLockError)?; + Ok(InMemoryAccountCursor::new(&inner, max_block_number)) + } + + fn ro_tx<'tx>(&'tx self) -> MorphProofsStorageResult> { + Ok(()) + } + + fn storage_trie_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.storage_trie_cursor(hashed_address, max_block_number) + } + + fn account_trie_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.account_trie_cursor(max_block_number) + } + + fn storage_hashed_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.storage_hashed_cursor(hashed_address, max_block_number) + } + + fn account_hashed_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.account_hashed_cursor(max_block_number) + } + + fn store_trie_updates( + &self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + let mut inner = self.inner.write(); + + Ok(inner.store_trie_updates(block_ref.block.number, block_state_diff)) + } + + fn fetch_trie_updates(&self, block_number: u64) -> MorphProofsStorageResult { + let inner = self.inner.read(); + + let trie_updates = inner + .trie_updates + .get(&block_number) + .cloned() + .unwrap_or_default(); + let post_state = inner + .post_states + .get(&block_number) + .cloned() + .unwrap_or_default(); + + Ok(BlockStateDiff { + sorted_trie_updates: trie_updates, + sorted_post_state: post_state, + }) + } + + fn prune_earliest_state( + &self, + new_earliest_block_ref: BlockWithParent, + ) -> MorphProofsStorageResult { + let mut write_counts = WriteCounts::default(); + let mut inner = self.inner.write(); + let new_earliest = new_earliest_block_ref.block.number; + + // 1. Account Branches + // Identify keys to move (blocks <= new_earliest but > 0) + let account_keys: Vec<_> = inner + .account_branches + .keys() + .filter(|(block, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in account_keys { + if let Some(branch) = inner.account_branches.remove(&key) { + // Determine if we should update the base state (block 0) + // In a simpler model without diff calculation, we just overwrite block 0 + // with the latest version found in the pruned range. + // Since keys are sorted by block, this logic naturally keeps the latest if we + // iterate in order, but here we are just grabbing all of them. + // For correctness in BTreeMap iteration order (which is sorted): + inner.account_branches.insert((0, key.1), branch); + write_counts.account_trie_updates_written_total += 1; + } + } + + // 2. Storage Branches + let storage_keys: Vec<_> = inner + .storage_branches + .keys() + .filter(|(block, _, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in storage_keys { + if let Some(branch) = inner.storage_branches.remove(&key) { + inner.storage_branches.insert((0, key.1, key.2), branch); + write_counts.storage_trie_updates_written_total += 1; + } + } + + // 3. Hashed Accounts + let acc_keys: Vec<_> = inner + .hashed_accounts + .keys() + .filter(|(block, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in acc_keys { + if let Some(acc) = inner.hashed_accounts.remove(&key) { + inner.hashed_accounts.insert((0, key.1), acc); + write_counts.hashed_accounts_written_total += 1; + } + } + + // 4. Hashed Storages + let stor_keys: Vec<_> = inner + .hashed_storages + .keys() + .filter(|(block, _, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in stor_keys { + if let Some(val) = inner.hashed_storages.remove(&key) { + inner.hashed_storages.insert((0, key.1, key.2), val); + write_counts.hashed_storages_written_total += 1; + } + } + + // Update earliest block pointer + if let Some((_, hash)) = inner.earliest_block { + inner.earliest_block = Some((new_earliest, hash)); + } + + // 5. Cleanup Metadata + inner.trie_updates.retain(|block, _| *block > new_earliest); + inner.post_states.retain(|block, _| *block > new_earliest); + + Ok(write_counts) + } + + fn unwind_history(&self, unwind_upto_block: BlockWithParent) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + let unwind_upto_block_number = unwind_upto_block.block.number - 1; + + // Remove all updates after unwind_upto_block_number + inner + .trie_updates + .retain(|block, _| *block <= unwind_upto_block_number); + inner + .post_states + .retain(|block, _| *block <= unwind_upto_block_number); + inner + .account_branches + .retain(|(block, _), _| *block <= unwind_upto_block_number); + inner + .storage_branches + .retain(|(block, _, _), _| *block <= unwind_upto_block_number); + inner + .hashed_accounts + .retain(|(block, _), _| *block <= unwind_upto_block_number); + inner + .hashed_storages + .retain(|(block, _, _), _| *block <= unwind_upto_block_number); + + Ok(()) + } + + fn replace_updates( + &self, + latest_common_block: BlockNumHash, + blocks_to_add: Vec<(BlockWithParent, BlockStateDiff)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + let latest_common_block_number = latest_common_block.number; + + // Remove all updates after latest_common_block_number + inner + .trie_updates + .retain(|block, _| *block <= latest_common_block_number); + inner + .post_states + .retain(|block, _| *block <= latest_common_block_number); + inner + .account_branches + .retain(|(block, _), _| *block <= latest_common_block_number); + inner + .storage_branches + .retain(|(block, _, _), _| *block <= latest_common_block_number); + inner + .hashed_accounts + .retain(|(block, _), _| *block <= latest_common_block_number); + inner + .hashed_storages + .retain(|(block, _, _), _| *block <= latest_common_block_number); + + for (block, block_state_diff) in blocks_to_add { + inner.store_trie_updates(block.block.number, block_state_diff); + } + + Ok(()) + } + + fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + inner.earliest_block = Some((block_number, hash)); + Ok(()) + } +} + +/// Degenerate batch session for [`InMemoryProofsStorage`]: writes go directly to shared +/// storage and are visible to subsequent reads. There is no commit/abort distinction — +/// the in-memory store does not provide cross-block atomicity. +#[derive(Debug)] +pub struct InMemoryBatchSession<'a> { + storage: &'a InMemoryProofsStorage, +} + +impl MorphProofsBatchSession for InMemoryBatchSession<'_> { + type StorageTrieCursor<'a> + = InMemoryTrieCursor + where + Self: 'a; + type AccountTrieCursor<'a> + = InMemoryTrieCursor + where + Self: 'a; + type StorageCursor<'a> + = InMemoryStorageCursor + where + Self: 'a; + type AccountHashedCursor<'a> + = InMemoryAccountCursor + where + Self: 'a; + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + self.storage.get_earliest_block_number() + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + self.storage.get_latest_block_number() + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage + .storage_trie_cursor(hashed_address, max_block_number) + } + + fn account_trie_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage.account_trie_cursor(max_block_number) + } + + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage + .storage_hashed_cursor(hashed_address, max_block_number) + } + + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage.account_hashed_cursor(max_block_number) + } + + fn store_trie_updates( + &mut self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + self.storage.store_trie_updates(block_ref, block_state_diff) + } +} + +impl MorphProofsBatchStore for InMemoryProofsStorage { + type BatchSession<'a> + = InMemoryBatchSession<'a> + where + Self: 'a; + + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult, + { + let mut session = InMemoryBatchSession { storage: self }; + f(&mut session) + } +} + +impl MorphProofsInitialStateStore for InMemoryProofsStorage { + fn initial_state_anchor(&self) -> MorphProofsStorageResult { + let inner = self.inner.read(); + + let Some((block_num, block_hash)) = inner.anchor_block else { + return Ok(InitialStateAnchor::default()); + }; + + let completed = inner.earliest_block.is_some(); + + Ok(InitialStateAnchor { + block: Some(NumHash::new(block_num, block_hash)), + status: if completed { + InitialStateStatus::Completed + } else { + InitialStateStatus::InProgress + }, + latest_account_trie_key: self.get_latest_account_trie_key()?, + latest_storage_trie_key: self.get_latest_storage_trie_key()?, + latest_hashed_account_key: self.get_latest_hashed_account_key()?, + latest_hashed_storage_key: self.get_latest_hashed_storage_key()?, + }) + } + + fn set_initial_state_anchor(&self, anchor: BlockNumHash) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + inner.anchor_block = Some((anchor.number, anchor.hash)); + Ok(()) + } + + fn store_account_branches( + &self, + updates: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (path, branch) in updates { + inner.account_branches.insert((0, path), branch); + } + + Ok(()) + } + + fn store_storage_branches( + &self, + hashed_address: B256, + items: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (path, branch) in items { + inner + .storage_branches + .insert((0, hashed_address, path), branch); + } + + Ok(()) + } + + fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (address, account) in accounts { + inner.hashed_accounts.insert((0, address), account); + } + + Ok(()) + } + + fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (slot, value) in storages { + inner + .hashed_storages + .insert((0, hashed_address, slot), value); + } + + Ok(()) + } + + fn commit_initial_state(&self) -> MorphProofsStorageResult { + let mut inner = self.inner.write(); + if let Some((number, hash)) = inner.anchor_block { + inner.earliest_block = Some((number, hash)); + Ok(BlockNumHash::new(number, hash)) + } else { + Err(MorphProofsStorageError::NoBlocksFound) + } + } +} + +#[cfg(test)] +mod tests { + use alloy_eips::NumHash; + use alloy_primitives::U256; + use reth_primitives_traits::Account; + + use super::*; + use crate::MorphProofsStorageError; + + #[test] + fn test_in_memory_storage_basic_operations() -> Result<(), MorphProofsStorageError> { + let storage = InMemoryProofsStorage::new(); + + // Test setting earliest block + let block_hash = B256::random(); + storage.set_earliest_block_number(1, block_hash)?; + let earliest = storage.get_earliest_block_number()?; + assert_eq!(earliest, Some((1, block_hash))); + + // Test storing and retrieving accounts + let account = Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }; + let hashed_address = B256::random(); + + storage.store_hashed_accounts(vec![(hashed_address, Some(account))])?; + + let _cursor = storage.account_hashed_cursor(10)?; + // Note: cursor testing would require more complex setup with proper seek/next operations + + Ok(()) + } + + #[test] + fn test_trie_updates_storage() -> Result<(), MorphProofsStorageError> { + let storage = InMemoryProofsStorage::new(); + + let sorted_trie_updates = TrieUpdatesSorted::default(); + let sorted_post_state = HashedPostStateSorted::default(); + let block_state_diff = BlockStateDiff { + sorted_trie_updates: sorted_trie_updates.clone(), + sorted_post_state: sorted_post_state.clone(), + }; + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(5, B256::ZERO)); + storage.store_trie_updates(BLOCK, block_state_diff)?; + + let retrieved_diff = storage.fetch_trie_updates(BLOCK.block.number)?; + assert_eq!(retrieved_diff.sorted_trie_updates, sorted_trie_updates); + assert_eq!(retrieved_diff.sorted_post_state, sorted_post_state); + + Ok(()) + } +} diff --git a/crates/proofs/src/initialize.rs b/crates/proofs/src/initialize.rs new file mode 100644 index 0000000..4f906df --- /dev/null +++ b/crates/proofs/src/initialize.rs @@ -0,0 +1,1374 @@ +//! Initialization job for proofs storage. Handles storing the existing state into the proofs +//! storage. + +use std::{collections::HashMap, time::Instant}; + +use alloy_eips::BlockNumHash; +use alloy_primitives::{B256, U256}; +use derive_more::Constructor; +use reth_db::{ + DatabaseError, + cursor::{DbCursorRO, DbDupCursorRO}, + tables, + transaction::DbTx, +}; +use reth_primitives_traits::{Account, StorageEntry}; +use reth_trie_common::{ + BranchNodeCompact, Nibbles, PackedStorageTrieEntry, PackedStoredNibbles, + PackedStoredNibblesSubKey, StoredNibbles, +}; +use tracing::{debug, info}; + +use crate::{ + MorphProofsStorageError, MorphProofsStore, + api::{InitialStateAnchor, InitialStateStatus, MorphProofsInitialStateStore}, + db::{HashedStorageKey, StorageTrieKey}, +}; + +/// Batch size threshold for storing entries during initialization +const INITIALIZE_STORAGE_THRESHOLD: usize = 100000; + +/// Threshold for logging progress during initialization +const INITIALIZE_LOG_THRESHOLD: usize = 100000; + +/// Initialization job for external storage backed by Reth Storage V2 tables. +#[derive(Debug, Constructor)] +pub struct InitializationJob { + storage: S, + tx: Tx, +} + +/// Macro to generate simple cursor iterators for tables +macro_rules! define_simple_cursor_iter { + ($iter_name:ident, $table:ty, $key_type:ty, $value_type:ty) => { + struct $iter_name(C); + + impl $iter_name { + const fn new(cursor: C) -> Self { + Self(cursor) + } + } + + impl> Iterator for $iter_name { + type Item = Result<($key_type, $value_type), DatabaseError>; + + fn next(&mut self) -> Option { + self.0.next().transpose() + } + } + }; +} + +/// Macro to generate duplicate cursor iterators for tables with custom logic +macro_rules! define_dup_cursor_iter { + ($iter_name:ident, $table:ty, $key_type:ty, $value_type:ty) => { + struct $iter_name(C); + + impl $iter_name { + const fn new(cursor: C) -> Self { + Self(cursor) + } + } + + impl + DbCursorRO<$table>> Iterator for $iter_name { + type Item = Result<($key_type, $value_type), DatabaseError>; + + fn next(&mut self) -> Option { + // First try to get the next duplicate value + if let Some(res) = self.0.next_dup().transpose() { + return Some(res); + } + + // If no more duplicates, find the next key with values + let Some(Ok((next_key, _))) = self.0.next_no_dup().transpose() else { + // If no more entries, return None + return None; + }; + + // If found, seek to the first duplicate for this key + return self.0.seek(next_key).transpose(); + } + } + }; +} + +// Generate iterators for the hashed-state and Storage V2 trie tables. +define_simple_cursor_iter!(HashedAccountsInit, tables::HashedAccounts, B256, Account); +define_dup_cursor_iter!( + HashedStoragesInit, + tables::HashedStorages, + B256, + StorageEntry +); +define_simple_cursor_iter!( + PackedAccountsTrieInit, + tables::PackedAccountsTrie, + PackedStoredNibbles, + BranchNodeCompact +); +define_dup_cursor_iter!( + PackedStoragesTrieInit, + tables::PackedStoragesTrie, + B256, + PackedStorageTrieEntry +); + +/// Trait to estimate the progress of a initialization job based on the key. +trait CompletionEstimatable { + // Returns a progress estimate as a percentage (0.0 to 1.0) + fn estimate_progress(&self) -> f64; +} + +impl CompletionEstimatable for B256 { + fn estimate_progress(&self) -> f64 { + // use the first 3 bytes as a progress estimate + let progress = self.0[..3].to_vec(); + let mut val: u64 = 0; + for nibble in &progress { + val = (val << 8) | *nibble as u64; + } + val as f64 / (256u64.pow(3)) as f64 + } +} + +impl CompletionEstimatable for Nibbles { + fn estimate_progress(&self) -> f64 { + // use the first 6 nibbles as a progress estimate + let progress_nibbles = if self.is_empty() { + Self::new() + } else { + self.slice(0..(self.len().min(6))) + }; + let mut val: u64 = 0; + for nibble in progress_nibbles.iter() { + val = (val << 4) | nibble as u64; + } + val as f64 / (16u64.pow(progress_nibbles.len() as u32)) as f64 + } +} + +impl CompletionEstimatable for PackedStoredNibbles { + fn estimate_progress(&self) -> f64 { + self.0.estimate_progress() + } +} + +impl + InitializationJob +{ + /// Initialize a table from a source iterator to a storage function. Handles batching and + /// logging. + fn initialize< + I: Iterator> + InitTable, + Key: CompletionEstimatable + 'static, + Value: 'static, + >( + &self, + name: &str, + source: I, + storage_threshold: usize, + log_threshold: usize, + ) -> Result { + info!(name = %name, "Starting initialization"); + let start_time = Instant::now(); + + let mut source = source.peekable(); + let Some(first_entry) = source.peek() else { + debug!(target: "reth::cli", "No entries to store for table"); + return Ok(0); + }; + let initial_progress = match first_entry { + Ok(i) => i.0.estimate_progress(), + Err(e) => Err(e.clone())?, + }; + + let storage = &self.storage; + let source_size_hint = source.size_hint().0; + let mut batch = Vec::with_capacity(source_size_hint.min(storage_threshold)); + let mut total_entries: usize = 0; + + for entry in source { + batch.push(entry?); + total_entries += 1; + + if total_entries.is_multiple_of(log_threshold) { + let progress = batch.last().expect("non-empty batch").0.estimate_progress(); + let elapsed = start_time.elapsed(); + let elapsed_secs = elapsed.as_secs_f64(); + + let progress_per_second = if elapsed_secs.is_normal() { + (progress - initial_progress) / elapsed_secs + } else { + 0.0 + }; + let estimated_total_time = if progress_per_second.is_normal() { + (1.0 - progress) / progress_per_second + } else { + 0.0 + }; + let progress_pct = progress * 100.0; + info!( + name = %name, + total_entries, + progress_pct = format_args!("{:.2}", progress_pct), + estimated_total_time = format_args!("{:.1}", estimated_total_time), + "Processed entries" + ); + } + + if batch.len() >= storage_threshold { + info!(name = %name, total_entries = total_entries, "Storing entries"); + I::store_entries(storage, batch)?; + batch = Vec::with_capacity( + (source_size_hint.saturating_sub(total_entries)).min(storage_threshold), + ); + } + } + + if !batch.is_empty() { + info!(name = %name, "Storing final entries"); + I::store_entries(storage, batch)?; + } + + info!(name = %name, total_entries = total_entries, "initialization complete"); + Ok(total_entries as u64) + } + + /// Initialize hashed accounts data + fn initialize_hashed_accounts( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_read::()?; + + if let Some(latest) = start_key { + start_cursor + .seek(latest)? + .filter(|(k, _)| *k == latest) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + let source = HashedAccountsInit::new(start_cursor); + self.initialize( + "hashed accounts", + source, + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Initialize hashed storage data + fn initialize_hashed_storages( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_dup_read::()?; + + if let Some(latest) = start_key { + start_cursor + .seek_by_key_subkey(latest.hashed_address, latest.hashed_storage_key)? + .filter(|v| v.key == latest.hashed_storage_key) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + let source = HashedStoragesInit::new(start_cursor); + self.initialize( + "hashed storage", + source, + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Initialize accounts trie data + fn initialize_accounts_trie( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_read::()?; + + if let Some(latest_key) = start_key { + let packed_key = PackedStoredNibbles::from(latest_key.0); + start_cursor + .seek(packed_key.clone())? + .filter(|(key, _)| *key == packed_key) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + self.initialize( + "accounts trie", + PackedAccountsTrieInit::new(start_cursor), + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Initialize storage trie data + fn initialize_storages_trie( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_dup_read::()?; + + if let Some(latest_key) = start_key { + let packed_subkey = PackedStoredNibblesSubKey::from(latest_key.path.0); + start_cursor + .seek_by_key_subkey(latest_key.hashed_address, packed_subkey)? + .filter(|value| value.nibbles.0 == latest_key.path.0) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + self.initialize( + "storage trie", + PackedStoragesTrieInit::new(start_cursor), + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Run complete initialization of all preimage data + fn initialize_trie(&self, anchor: InitialStateAnchor) -> Result<(), MorphProofsStorageError> { + self.initialize_hashed_accounts(anchor.latest_hashed_account_key)?; + self.initialize_hashed_storages(anchor.latest_hashed_storage_key)?; + self.initialize_storages_trie(anchor.latest_storage_trie_key)?; + self.initialize_accounts_trie(anchor.latest_account_trie_key)?; + Ok(()) + } + + fn validate_anchor_block( + &self, + anchor: &InitialStateAnchor, + best_number: u64, + best_hash: B256, + ) -> Result<(), MorphProofsStorageError> { + let block = anchor + .block + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + + if block.number != best_number || block.hash != best_hash { + return Err(MorphProofsStorageError::InitializeStorageInconsistentState); + } + + Ok(()) + } + + /// Run the initialization job. + pub fn run(&self, best_number: u64, best_hash: B256) -> Result<(), MorphProofsStorageError> { + let anchor = self.storage.initial_state_anchor()?; + + match anchor.status { + InitialStateStatus::Completed => return Ok(()), + InitialStateStatus::NotStarted => { + self.storage + .set_initial_state_anchor(BlockNumHash::new(best_number, best_hash))?; + } + InitialStateStatus::InProgress => { + self.validate_anchor_block(&anchor, best_number, best_hash)?; + } + } + + self.initialize_trie(anchor)?; + self.storage.commit_initial_state()?; + + Ok(()) + } +} + +/// Handles storing entries for a particular KV-pair type. +trait InitTable { + /// Key of target table. + type Key: CompletionEstimatable + 'static; + /// Value of target table. + type Value: 'static; + + /// Writes given entries to given storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError>; +} + +impl InitTable for HashedAccountsInit { + type Key = B256; + type Value = Account; + + /// Save mapping of hashed addresses to accounts to storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + store.store_hashed_accounts( + entries + .into_iter() + .map(|(address, account)| (address, Some(account))) + .collect(), + )?; + Ok(()) + } +} + +impl InitTable for HashedStoragesInit { + type Key = B256; + type Value = StorageEntry; + + /// Save mapping of hashed addresses to storage entries to storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + let entries_iter = entries.into_iter(); + let mut by_address: HashMap> = + HashMap::with_capacity(entries_iter.size_hint().0); + + for (address, entry) in entries_iter { + by_address + .entry(address) + .or_default() + .push((entry.key, entry.value)); + } + store.store_hashed_storages_bulk(by_address.into_iter().collect())?; + + Ok(()) + } +} + +impl InitTable for PackedAccountsTrieInit { + type Key = PackedStoredNibbles; + type Value = BranchNodeCompact; + + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + store.store_account_branches( + entries + .into_iter() + .map(|(path, branch)| (path.0, Some(branch))) + .collect(), + )?; + Ok(()) + } +} + +impl InitTable for PackedStoragesTrieInit { + type Key = B256; + type Value = PackedStorageTrieEntry; + + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + let entries_iter = entries.into_iter(); + let mut by_address: HashMap)>> = + HashMap::with_capacity(entries_iter.size_hint().0); + + for (hashed_address, storage_entry) in entries_iter { + by_address + .entry(hashed_address) + .or_default() + .push((storage_entry.nibbles.0, Some(storage_entry.node))); + } + store.store_storage_branches_bulk(by_address.into_iter().collect())?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + macro_rules! proof_storage_init_tests { + ($mod_name:ident, $storage:ident) => { + mod $mod_name { + use std::sync::Arc; + + use alloy_primitives::{Address, U256, keccak256}; + use reth_db::{ + Database, cursor::DbCursorRW, test_utils::create_test_rw_db, + transaction::DbTxMut, + }; + use reth_primitives_traits::Account; + use reth_trie::{ + BranchNodeCompact, PackedStorageTrieEntry, PackedStoredNibbles, + PackedStoredNibblesSubKey, TrieMask, hashed_cursor::HashedCursor, + trie_cursor::TrieCursor, + }; + use tempfile::TempDir; + + use super::super::*; + use crate::$storage; + + /// Helper function to create a key + fn k(b: u8) -> B256 { + let mut bytes = [0u8; 32]; + bytes[0] = b; + B256::from(bytes) + } + + /// Helper function to create a test branch node + fn create_test_branch_node() -> BranchNodeCompact { + let mut state_mask = TrieMask::default(); + state_mask.set_bit(0); + state_mask.set_bit(1); + + BranchNodeCompact { + state_mask, + tree_mask: TrieMask::default(), + hash_mask: TrieMask::default(), + hashes: Arc::new(vec![]), + root_hash: None, + } + } + + #[test] + fn test_initialize_hashed_accounts() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test accounts into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); + + let mut accounts = vec![ + ( + keccak256(Address::repeat_byte(0x01)), + Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }, + ), + ( + keccak256(Address::repeat_byte(0x02)), + Account { + nonce: 2, + balance: U256::from(200), + bytecode_hash: None, + }, + ), + ( + keccak256(Address::repeat_byte(0x03)), + Account { + nonce: 3, + balance: U256::from(300), + bytecode_hash: None, + }, + ), + ]; + + // Sort accounts by address for cursor.append (which requires sorted order) + accounts.sort_by_key(|(addr, _)| *addr); + + for (addr, account) in &accounts { + cursor.append(*addr, account).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_hashed_accounts(None).unwrap(); + + // Verify data was stored (will be in sorted order) + let mut account_cursor = storage.account_hashed_cursor(100).unwrap(); + let mut count = 0; + while let Some((key, account)) = account_cursor.next().unwrap() { + // Find matching account in our test data + let expected = accounts.iter().find(|(addr, _)| *addr == key).unwrap(); + assert_eq!((key, account), *expected); + count += 1; + } + assert_eq!(count, 3); + } + + #[test] + fn test_initialize_hashed_storage() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test storage into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); + + let addr1 = keccak256(Address::repeat_byte(0x01)); + let addr2 = keccak256(Address::repeat_byte(0x02)); + + let storage_entries = vec![ + ( + addr1, + StorageEntry { + key: keccak256(B256::repeat_byte(0x10)), + value: U256::from(100), + }, + ), + ( + addr1, + StorageEntry { + key: keccak256(B256::repeat_byte(0x20)), + value: U256::from(200), + }, + ), + ( + addr2, + StorageEntry { + key: keccak256(B256::repeat_byte(0x30)), + value: U256::from(300), + }, + ), + ]; + + for (addr, entry) in &storage_entries { + cursor.upsert(*addr, entry).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_hashed_storages(None).unwrap(); + + // Verify data was stored for addr1 + let mut storage_cursor = storage.storage_hashed_cursor(addr1, 100).unwrap(); + let mut found = vec![]; + while let Some((key, value)) = storage_cursor.next().unwrap() { + found.push((key, value)); + } + assert_eq!(found.len(), 2); + assert_eq!( + found[0], + (storage_entries[0].1.key, storage_entries[0].1.value) + ); + assert_eq!( + found[1], + (storage_entries[1].1.key, storage_entries[1].1.value) + ); + + // Verify data was stored for addr2 + let mut storage_cursor = storage.storage_hashed_cursor(addr2, 100).unwrap(); + let mut found = vec![]; + while let Some((key, value)) = storage_cursor.next().unwrap() { + found.push((key, value)); + } + assert_eq!(found.len(), 1); + assert_eq!( + found[0], + (storage_entries[2].1.key, storage_entries[2].1.value) + ); + } + + #[test] + fn test_initialize_accounts_trie() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test trie nodes into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); + + let branch = create_test_branch_node(); + let nodes = vec![ + ( + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![1])), + branch.clone(), + ), + ( + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![2])), + branch.clone(), + ), + ( + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![3])), + branch, + ), + ]; + + for (path, node) in &nodes { + cursor.append(path.clone(), node).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_accounts_trie(None).unwrap(); + + // Verify data was stored + let mut trie_cursor = storage.account_trie_cursor(100).unwrap(); + let mut count = 0; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + assert_eq!(path, nodes[count].0.0); + count += 1; + } + assert_eq!(count, 3); + } + + #[test] + fn test_initialize_storages_trie() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test storage trie nodes into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); + + let branch = create_test_branch_node(); + let addr1 = keccak256(Address::repeat_byte(0x01)); + let addr2 = keccak256(Address::repeat_byte(0x02)); + + let nodes = vec![ + ( + addr1, + PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![1]), + ), + node: branch.clone(), + }, + ), + ( + addr1, + PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![2]), + ), + node: branch.clone(), + }, + ), + ( + addr2, + PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![3]), + ), + node: branch, + }, + ), + ]; + + for (addr, entry) in &nodes { + cursor.upsert(*addr, entry).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_storages_trie(None).unwrap(); + + // Verify data was stored for addr1 + let mut trie_cursor = storage.storage_trie_cursor(addr1, 100).unwrap(); + let mut found = vec![]; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + found.push(path); + } + assert_eq!(found.len(), 2); + assert_eq!(found[0], nodes[0].1.nibbles.0); + assert_eq!(found[1], nodes[1].1.nibbles.0); + + // Verify data was stored for addr2 + let mut trie_cursor = storage.storage_trie_cursor(addr2, 100).unwrap(); + let mut found = vec![]; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + found.push(path); + } + assert_eq!(found.len(), 1); + assert_eq!(found[0], nodes[2].1.nibbles.0); + } + + #[test] + fn test_full_initialize_run() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert some test data + let tx = db.tx_mut().unwrap(); + + // Add accounts + let mut cursor = tx.cursor_write::().unwrap(); + let addr = keccak256(Address::repeat_byte(0x01)); + cursor + .append( + addr, + &Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }, + ) + .unwrap(); + drop(cursor); + + // Add storage + let mut cursor = tx.cursor_dup_write::().unwrap(); + cursor + .upsert( + addr, + &StorageEntry { + key: keccak256(B256::repeat_byte(0x10)), + value: U256::from(100), + }, + ) + .unwrap(); + drop(cursor); + + // Add account trie + let mut cursor = tx.cursor_write::().unwrap(); + cursor + .append( + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![1])), + &create_test_branch_node(), + ) + .unwrap(); + drop(cursor); + + // Add storage trie + let mut cursor = tx.cursor_dup_write::().unwrap(); + cursor + .upsert( + addr, + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![1]), + ), + node: create_test_branch_node(), + }, + ) + .unwrap(); + drop(cursor); + + tx.commit().unwrap(); + + // Run full initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + let best_number = 100; + let best_hash = B256::repeat_byte(0x42); + + // Should be None initially + assert_eq!(storage.initial_state_anchor().unwrap().block, None); + assert_eq!(storage.get_earliest_block_number().unwrap(), None); + + job.run(best_number, best_hash).unwrap(); + + // Should be set after initialization + assert_eq!( + storage.get_earliest_block_number().unwrap(), + Some((best_number, best_hash)) + ); + + // Verify data was initialized + let mut account_cursor = storage.account_hashed_cursor(100).unwrap(); + assert!(account_cursor.next().unwrap().is_some()); + + let mut storage_cursor = storage.storage_hashed_cursor(addr, 100).unwrap(); + assert!(storage_cursor.next().unwrap().is_some()); + + let mut trie_cursor = storage.account_trie_cursor(100).unwrap(); + assert!(trie_cursor.next().unwrap().is_some()); + + let mut storage_trie_cursor = storage.storage_trie_cursor(addr, 100).unwrap(); + assert!(storage_trie_cursor.next().unwrap().is_some()); + } + + #[test] + fn test_initialize_run_skips_if_already_done() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // set and commit initial state anchor + storage + .set_initial_state_anchor(BlockNumHash::new(50, B256::repeat_byte(0x01))) + .expect("set anchor"); + storage.commit_initial_state().expect("commit anchor"); + + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + + // Run initialization - should skip + job.run(100, B256::repeat_byte(0x42)).unwrap(); + + // Should still have the old anchor + let anchor_block = storage + .initial_state_anchor() + .expect("get anchor") + .block + .expect("block"); + assert_eq!( + Some((anchor_block.number, anchor_block.hash)), + Some((50, B256::repeat_byte(0x01))) + ); + + // Should still have the old earliest block + assert_eq!( + storage.get_earliest_block_number().unwrap(), + Some((50, B256::repeat_byte(0x01))) + ); + } + + #[test] + fn test_initialize_resumes_hashed_accounts_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + // Phase 1 in source: k1, k2 + let k1 = k(1); + let k2 = k(2); + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + k1, + &Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }, + ) + .unwrap(); + cur.append( + k2, + &Account { + nonce: 2, + balance: U256::from(200), + bytecode_hash: None, + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_accounts(None).unwrap(); + } + + // Resume point must be k2 (max) + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_account_key, + Some(k2) + ); + + // Phase 2 in source: k3, k4 + let k3 = k(3); + let k4 = k(4); + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + k3, + &Account { + nonce: 3, + balance: U256::from(300), + bytecode_hash: None, + }, + ) + .unwrap(); + cur.append( + k4, + &Account { + nonce: 4, + balance: U256::from(400), + bytecode_hash: None, + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 (restart) + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_accounts(Some(k2)).unwrap(); + } + + // Now resume point must be k4 + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_account_key, + Some(k4) + ); + + // Verify order + no dupes by iterating proofs cursor + let mut cur = store.account_hashed_cursor(0).unwrap(); + let mut got = Vec::new(); + while let Some((key, acct)) = cur.next().unwrap() { + got.push((key, acct)); + } + + // Expect exactly 4, in increasing key order. + assert_eq!(got.len(), 4); + assert_eq!(got[0].0, k1); + assert_eq!(got[1].0, k2); + assert_eq!(got[2].0, k3); + assert_eq!(got[3].0, k4); + + // No dupes + for w in got.windows(2) { + assert!(w[0].0 < w[1].0); + } + } + + #[test] + fn test_initialize_resumes_hashed_storages_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + let a1 = k(0x10); + let a2 = k(0x20); + + let s11 = k(0x01); + let s12 = k(0x02); + let s21 = k(0x03); + let s22 = k(0x04); + + // Phase 1 source: + // a1: s11,s12 + // a2: s21 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a1, + &StorageEntry { + key: s11, + value: U256::from(11), + }, + ) + .unwrap(); + cur.upsert( + a1, + &StorageEntry { + key: s12, + value: U256::from(12), + }, + ) + .unwrap(); + cur.upsert( + a2, + &StorageEntry { + key: s21, + value: U256::from(21), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_storages(None).unwrap(); + } + + // Latest key must be (a2, s21) because a2 > a1 + let last1 = store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_storage_key + .expect("ok"); + assert_eq!(last1.hashed_address, a2); + assert_eq!(last1.hashed_storage_key, s21); + + // Phase 2 source: add s22 to a2 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a2, + &StorageEntry { + key: s22, + value: U256::from(22), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_storages(Some(HashedStorageKey::new(a2, s21))) + .unwrap(); + } + + // Latest key now must be (a2, s22) + let last2 = store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_storage_key + .expect("ok"); + assert_eq!(last2.hashed_address, a2); + assert_eq!(last2.hashed_storage_key, s22); + + // Verify no dupes by iterating per-address + { + let mut c = store.storage_hashed_cursor(a1, 0).unwrap(); + let mut got = Vec::new(); + while let Some((slot, val)) = c.next().unwrap() { + got.push((slot, val)); + } + assert_eq!(got.len(), 2); + assert_eq!(got[0].0, s11); + assert_eq!(got[1].0, s12); + } + { + let mut c = store.storage_hashed_cursor(a2, 0).unwrap(); + let mut got = Vec::new(); + while let Some((slot, val)) = c.next().unwrap() { + got.push((slot, val)); + } + assert_eq!(got.len(), 2); + assert_eq!(got[0].0, s21); + assert_eq!(got[1].0, s22); + } + } + + #[test] + fn test_initialize_resumes_accounts_trie_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + let p1 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])); + let p2 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![2])); + let p3 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![3])); + let p4 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![4])); + + // Phase 1 source: p1,p2 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + PackedStoredNibbles::from(p1.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); + cur.append( + PackedStoredNibbles::from(p2.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_accounts_trie(None).unwrap(); + } + + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_account_trie_key, + Some(p2.clone()) + ); + + // Phase 2 source: p3,p4 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + PackedStoredNibbles::from(p3.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); + cur.append( + PackedStoredNibbles::from(p4.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_accounts_trie(Some(p2.clone())).unwrap(); + } + + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_account_trie_key, + Some(p4.clone()) + ); + + // Verify 4 ordered, no dupes + let mut c = store.account_trie_cursor(0).unwrap(); + let mut got = Vec::new(); + while let Some((path, _node)) = c.next().unwrap() { + got.push(path); + } + assert_eq!(got.len(), 4); + assert_eq!(got[0], p1.0); + assert_eq!(got[1], p2.0); + assert_eq!(got[2], p3.0); + assert_eq!(got[3], p4.0); + } + + #[test] + fn test_initialize_resumes_storages_trie_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + let a1 = k(0x10); + let a2 = k(0x20); + + let n1 = Nibbles::from_nibbles_unchecked(vec![1]); + let n2 = Nibbles::from_nibbles_unchecked(vec![2]); + let n3 = Nibbles::from_nibbles_unchecked(vec![3]); + + // Phase 1 source: (a1,n1), (a2,n2) + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a1, + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from(n1.clone()), + node: create_test_branch_node(), + }, + ) + .unwrap(); + cur.upsert( + a2, + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from(n2.clone()), + node: create_test_branch_node(), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_storages_trie(None).unwrap(); + } + + // Latest must be (a2, n2) because a2 > a1 + let last1 = store + .initial_state_anchor() + .expect("get anchor") + .latest_storage_trie_key + .expect("ok"); + assert_eq!(last1.hashed_address, a2); + assert_eq!(last1.path.0, n2); + + // Phase 2 source: add (a2,n3) + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a2, + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from(n3.clone()), + node: create_test_branch_node(), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_storages_trie(Some(StorageTrieKey::new( + a2, + StoredNibbles::from(n2.clone()), + ))) + .unwrap(); + } + + // Latest must now be (a2,n3) + let last2 = store + .initial_state_anchor() + .expect("get anchor") + .latest_storage_trie_key + .expect("ok"); + assert_eq!(last2.hashed_address, a2); + assert_eq!(last2.path.0, n3); + + // Verify per-address no dupes and stable ordering + { + let mut c = store.storage_trie_cursor(a1, 0).unwrap(); + + let mut got = Vec::new(); + + // next returns the rest + while let Some((path, _node)) = c.next().unwrap() { + got.push(path); + } + + assert_eq!(got, vec![n1]); + } + { + let mut c = store.storage_trie_cursor(a2, 0).unwrap(); + + let mut got = Vec::new(); + + // next returns the rest + while let Some((path, _node)) = c.next().unwrap() { + got.push(path); + } + assert_eq!(got, vec![n2, n3]); + } + } + } + }; + } + + proof_storage_init_tests!(mdbx_tests, MdbxProofsStorage); +} diff --git a/crates/proofs/src/lib.rs b/crates/proofs/src/lib.rs new file mode 100644 index 0000000..953632d --- /dev/null +++ b/crates/proofs/src/lib.rs @@ -0,0 +1,64 @@ +//! Bounded historical MPT storage for Morph `eth_getProof`. +//! +//! The versioned-trie implementation is adapted from Base's proof-history +//! storage at commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` under MIT. +//! Morph intentionally exposes only the MDBX production backend. + +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +pub mod api; +pub use api::{ + BlockStateDiff, InitialStateAnchor, InitialStateStatus, MorphProofsBatchSession, + MorphProofsBatchStore, MorphProofsInitialStateStore, MorphProofsStore, OperationDurations, + StorageBranchEntries, WriteCounts, +}; + +pub mod initialize; +pub use initialize::InitializationJob; + +/// Default proof-history retention window: 7 days at a one-second block time. +pub const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 604_800; + +#[cfg(any(test, feature = "test-utils"))] +pub mod in_memory; +#[cfg(any(test, feature = "test-utils"))] +pub use in_memory::{ + InMemoryAccountCursor, InMemoryBatchSession, InMemoryProofsStorage, InMemoryStorageCursor, + InMemoryTrieCursor, +}; + +pub mod db; +pub use db::{ + MdbxAccountCursor, MdbxBatchSession, MdbxProofsStorage, MdbxProofsStorageOptions, + MdbxStorageCursor, MdbxTrieCursor, ProofDbIdentity, ProofWindowBounds, +}; + +pub mod metrics; +pub use metrics::{ + MorphProofsHashedAccountCursor, MorphProofsHashedStorageCursor, MorphProofsStorage, + MorphProofsTrieCursor, ProofHistoryMetrics, +}; + +pub mod proof; +pub mod provider; + +mod batch_provider; +pub use batch_provider::MorphProofsBatchStateProviderRef; + +pub mod live; + +pub mod cursor; +pub mod cursor_factory; +pub use cursor_factory::{ + MorphProofsBatchHashedAccountCursorFactory, MorphProofsBatchTrieCursorFactory, + MorphProofsHashedAccountCursorFactory, MorphProofsTrieCursorFactory, +}; + +pub mod error; +pub use error::{MorphProofsStorageError, MorphProofsStorageResult}; + +mod prune; +pub use prune::{ + MorphProofStoragePruner, MorphProofStoragePrunerResult, MorphProofStoragePrunerTask, + PrunerError, PrunerOutput, +}; diff --git a/crates/proofs/src/live.rs b/crates/proofs/src/live.rs new file mode 100644 index 0000000..fafc102 --- /dev/null +++ b/crates/proofs/src/live.rs @@ -0,0 +1,432 @@ +//! Live trie collector for external proofs storage. + +use std::{sync::Arc, time::Instant}; + +use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; +use derive_more::Constructor; +use reth_evm::{ConfigureEvm, execute::Executor}; +use reth_primitives_traits::{AlloyBlockHeader, BlockTy, NodePrimitives, RecoveredBlock}; +use reth_provider::{ + DatabaseProviderFactory, HashedPostStateProvider, StateProviderFactory, StateRootProvider, +}; +use reth_revm::database::StateProviderDatabase; +use reth_trie_common::{HashedPostStateSorted, updates::TrieUpdatesSorted}; +use tracing::{info, warn}; + +use crate::{ + BlockStateDiff, MorphProofsStorage, MorphProofsStorageError, MorphProofsStore, + api::{MorphProofsBatchSession, MorphProofsBatchStore, OperationDurations, WriteCounts}, + batch_provider::MorphProofsBatchStateProviderRef, + metrics::ProofHistoryMetrics, + provider::MorphProofsStateProviderRef, +}; + +/// Live trie collector for external proofs storage. +#[derive(Debug, Constructor)] +pub struct LiveTrieCollector<'tx, Evm, Provider, PreimageStore> +where + Evm: ConfigureEvm, + Provider: DatabaseProviderFactory + StateProviderFactory, +{ + evm_config: Evm, + provider: Provider, + storage: &'tx MorphProofsStorage, +} + +impl<'tx, Evm, Provider, Store> LiveTrieCollector<'tx, Evm, Provider, Store> +where + Evm: ConfigureEvm, + Provider: DatabaseProviderFactory + StateProviderFactory, + Store: 'tx + MorphProofsStore + Clone + 'static, +{ + fn record_storage_metrics( + operation_durations: &OperationDurations, + write_counts: Option<&WriteCounts>, + ) { + ProofHistoryMetrics::record_block(operation_durations, write_counts); + } + + /// Execute a block and store the updates in the storage. + pub fn execute_and_store_block_updates( + &self, + block: &RecoveredBlock>, + ) -> Result<(), MorphProofsStorageError> { + let mut operation_durations = OperationDurations::default(); + + let start = Instant::now(); + // ensure that we have the state of the parent block + let (Some((earliest, _)), Some((latest, _))) = ( + self.storage.get_earliest_block_number()?, + self.storage.get_latest_block_number()?, + ) else { + return Err(MorphProofsStorageError::NoBlocksFound); + }; + + let parent_block_number = block.number() - 1; + if parent_block_number < earliest { + return Err(MorphProofsStorageError::UnknownParent); + } + + if parent_block_number > latest { + return Err(MorphProofsStorageError::MissingParentBlock { + block_number: block.number(), + parent_block_number, + latest_block_number: latest, + }); + } + + let block_ref = BlockWithParent::new( + block.parent_hash(), + NumHash::new(block.number(), block.hash()), + ); + + let state_provider = MorphProofsStateProviderRef::new( + self.provider.latest()?, + self.storage, + parent_block_number, + ); + + let db = StateProviderDatabase::new(&state_provider); + let block_executor = self.evm_config.batch_executor(db); + + let execution_result = block_executor.execute(&(*block).clone())?; + + operation_durations.execution_duration_seconds = start.elapsed(); + + let hashed_state = state_provider.hashed_post_state(&execution_result.state); + let (state_root, trie_updates) = + state_provider.state_root_with_updates(hashed_state.clone())?; + + operation_durations.state_root_duration_seconds = + start.elapsed() - operation_durations.execution_duration_seconds; + + if state_root != block.state_root() { + return Err(MorphProofsStorageError::StateRootMismatch { + block_number: block.number(), + current_state_hash: state_root, + expected_state_hash: block.state_root(), + }); + } + + let update_result = self.storage.store_trie_updates( + block_ref, + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: hashed_state.into_sorted(), + }, + )?; + + operation_durations.total_duration_seconds = start.elapsed(); + operation_durations.write_duration_seconds = operation_durations.total_duration_seconds + - operation_durations.state_root_duration_seconds + - operation_durations.execution_duration_seconds; + + Self::record_storage_metrics(&operation_durations, Some(&update_result)); + + info!( + block_number = block.number(), + block_hash = ?block.hash(), + parent_block_number, + parent_block_hash = ?block.parent_hash(), + earliest_proof_before = earliest, + latest_proof_before = latest, + latest_proof_after = block.number(), + state_root = ?state_root, + ?operation_durations, + ?update_result, + "Block executed and trie updates stored successfully", + ); + + Ok(()) + } + + /// Store trie updates for a given block. + pub fn store_block_updates( + &self, + block: BlockWithParent, + sorted_trie_updates: TrieUpdatesSorted, + sorted_post_state: HashedPostStateSorted, + ) -> Result<(), MorphProofsStorageError> { + let start = Instant::now(); + let mut operation_durations = OperationDurations::default(); + + let storage_result = match self.storage.store_trie_updates( + block, + BlockStateDiff { + sorted_trie_updates, + sorted_post_state, + }, + ) { + Ok(res) => res, + Err(MorphProofsStorageError::OutOfOrder { + block_number, + latest_block_hash, + parent_block_hash, + }) => { + warn!( + block_number, + ?latest_block_hash, + ?parent_block_hash, + "Skipping out of order block updates" + ); + return Ok(()); + } + Err(e) => return Err(e), + }; + + let write_duration = start.elapsed(); + operation_durations.total_duration_seconds = write_duration; + operation_durations.write_duration_seconds = write_duration; + + Self::record_storage_metrics(&operation_durations, Some(&storage_result)); + + info!( + block_number = block.block.number, + ?operation_durations, + ?storage_result, + "Trie updates stored successfully", + ); + + Ok(()) + } + + /// Handles chain reorganizations by replacing block updates after a common ancestor. + /// + /// This method removes all block updates after the latest common ancestor (the block before + /// the first block in `new_blocks`) and replaces them with the updates from the provided new + /// chain. + /// + /// # Arguments + /// + /// * `new_blocks` - A vector of references to `RecoveredBlock` instances representing the new + /// blocks to be added to the trie storage. + pub fn unwind_and_store_block_updates( + &self, + block_updates: Vec<( + BlockWithParent, + Arc, + Arc, + )>, + ) -> Result<(), MorphProofsStorageError> { + if block_updates.is_empty() { + return Ok(()); + } + + let start = Instant::now(); + let mut operation_durations = OperationDurations::default(); + let first = &block_updates[0].0; + let latest_common_block = + BlockNumHash::new(first.block.number.saturating_sub(1), first.parent); + let mut block_trie_updates: Vec<(BlockWithParent, BlockStateDiff)> = + Vec::with_capacity(block_updates.len()); + + for (block, trie_updates, hashed_state) in &block_updates { + block_trie_updates.push(( + *block, + BlockStateDiff { + sorted_trie_updates: (**trie_updates).clone(), + sorted_post_state: (**hashed_state).clone(), + }, + )); + } + + self.storage + .replace_updates(latest_common_block, block_trie_updates)?; + let write_duration = start.elapsed(); + operation_durations.total_duration_seconds = write_duration; + operation_durations.write_duration_seconds = write_duration; + + Self::record_storage_metrics(&operation_durations, None); + + info!( + start_block_number = block_updates.first().map(|(b, _, _)| b.block.number), + end_block_number = block_updates.last().map(|(b, _, _)| b.block.number), + ?operation_durations, + "Trie updates rewound and stored successfully", + ); + Ok(()) + } + + /// Remove account, storage and trie updates from historical storage for all blocks from + /// the specified block (inclusive). + pub fn unwind_history(&self, to: BlockWithParent) -> Result<(), MorphProofsStorageError> { + if let Err(error) = self.storage.unwind_history(to) { + ProofHistoryMetrics::record_unwind_error(); + return Err(error); + } + if let Some((earliest, _)) = self.storage.get_earliest_block_number()? + && let Some((latest, _)) = self.storage.get_latest_block_number()? + { + ProofHistoryMetrics::set_window(earliest, latest); + } + Ok(()) + } +} + +/// One block to process inside a batch session: either pre-computed cached trie data (fast +/// path) or a fully recovered block that needs full execution against the session's +/// transaction-local state (cold catch-up path). +#[derive(Debug)] +pub enum BatchBlock { + /// Pre-computed cached trie data; only writes happen. + Cached { + /// Block reference being written. + block_with_parent: BlockWithParent, + /// Pre-computed trie updates from cached notification data. + sorted_trie_updates: Arc, + /// Pre-computed hashed post-state from cached notification data. + sorted_post_state: Arc, + }, + /// Full block requiring execution against session-local state. + Execute(Box>>), +} + +impl<'tx, Evm, Provider, Store> LiveTrieCollector<'tx, Evm, Provider, Store> +where + Evm: ConfigureEvm, + Provider: DatabaseProviderFactory + StateProviderFactory, + Store: 'tx + MorphProofsBatchStore + Clone + 'static, +{ + /// Execute and write a batch of blocks inside a single underlying transaction. + /// Reads during execution of block N observe writes from blocks earlier in the batch, + /// enabling cold catch-up where parent state is staged but not yet committed. The + /// entire batch commits atomically on success and aborts on the first error. + pub fn execute_and_store_batch( + &self, + blocks: Vec>, + ) -> Result<(), MorphProofsStorageError> { + if blocks.is_empty() { + return Ok(()); + } + + let start = Instant::now(); + let mut total_writes = WriteCounts::default(); + let mut block_count: u64 = 0; + let mut last_block_number: u64 = 0; + + self.storage.with_batch_session(|session| { + let (Some((earliest, _)), Some((_, _))) = ( + session.get_earliest_block_number()?, + session.get_latest_block_number()?, + ) else { + return Err(MorphProofsStorageError::NoBlocksFound); + }; + + for entry in blocks { + match entry { + BatchBlock::Cached { + block_with_parent, + sorted_trie_updates, + sorted_post_state, + } => { + let counts = session.store_trie_updates( + block_with_parent, + BlockStateDiff { + sorted_trie_updates: (*sorted_trie_updates).clone(), + sorted_post_state: (*sorted_post_state).clone(), + }, + )?; + total_writes += counts; + block_count += 1; + last_block_number = block_with_parent.block.number; + } + BatchBlock::Execute(block) => { + let counts = self.execute_one_in_session(session, &block, earliest)?; + total_writes += counts; + block_count += 1; + last_block_number = block.number(); + } + } + } + + Ok(()) + })?; + + let total = start.elapsed(); + ProofHistoryMetrics::record_block( + &OperationDurations { + total_duration_seconds: total, + ..Default::default() + }, + Some(&total_writes), + ); + if let Some((earliest, _)) = self.storage.get_earliest_block_number()? { + ProofHistoryMetrics::set_window(earliest, last_block_number); + } + + info!( + block_count, + last_block_number, + ?total_writes, + duration = ?total, + "Batch executed and committed", + ); + + Ok(()) + } + + fn execute_one_in_session( + &self, + session: &mut S, + block: &RecoveredBlock>, + earliest: u64, + ) -> Result + where + S: MorphProofsBatchSession, + { + let latest_in_session = session + .get_latest_block_number()? + .ok_or(MorphProofsStorageError::NoBlocksFound)? + .0; + + let parent_block_number = block.number() - 1; + if parent_block_number < earliest { + return Err(MorphProofsStorageError::UnknownParent); + } + if parent_block_number > latest_in_session { + return Err(MorphProofsStorageError::MissingParentBlock { + block_number: block.number(), + parent_block_number, + latest_block_number: latest_in_session, + }); + } + + let block_ref = BlockWithParent::new( + block.parent_hash(), + NumHash::new(block.number(), block.hash()), + ); + + let state_provider = MorphProofsBatchStateProviderRef::new( + self.provider.latest()?, + session, + parent_block_number, + ); + + let db = StateProviderDatabase::new(&state_provider); + let block_executor = self.evm_config.batch_executor(db); + + let execution_result = block_executor.execute(&(*block).clone())?; + + let hashed_state = state_provider.hashed_post_state(&execution_result.state); + let (state_root, trie_updates) = + state_provider.state_root_with_updates(hashed_state.clone())?; + + if state_root != block.state_root() { + return Err(MorphProofsStorageError::StateRootMismatch { + block_number: block.number(), + current_state_hash: state_root, + expected_state_hash: block.state_root(), + }); + } + + drop(state_provider); + + session.store_trie_updates( + block_ref, + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: hashed_state.into_sorted(), + }, + ) + } +} diff --git a/crates/proofs/src/metrics.rs b/crates/proofs/src/metrics.rs new file mode 100644 index 0000000..21211c9 --- /dev/null +++ b/crates/proofs/src/metrics.rs @@ -0,0 +1,97 @@ +//! Metrics for bounded historical-proof storage. + +use reth_metrics::{ + Metrics, + metrics::{Counter, Gauge, Histogram}, +}; + +use crate::{OperationDurations, WriteCounts, cursor}; + +/// The production storage type. Kept as an alias so callers remain backend-agnostic +/// without introducing another production backend. +pub type MorphProofsStorage = S; + +/// Trie cursor used by proof generation. +pub type MorphProofsTrieCursor = cursor::MorphProofsTrieCursor; +/// Hashed account cursor used by proof generation. +pub type MorphProofsHashedAccountCursor = cursor::MorphProofsHashedAccountCursor; +/// Hashed storage cursor used by proof generation. +pub type MorphProofsHashedStorageCursor = cursor::MorphProofsHashedStorageCursor; + +/// Proof-history storage and lifecycle metrics. +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.proofs")] +pub struct ProofHistoryMetrics { + /// Total time spent processing a block. + pub block_total_duration_seconds: Histogram, + /// Time spent executing a block when catch-up re-execution is required. + pub block_execution_duration_seconds: Histogram, + /// Time spent calculating a state root. + pub block_state_root_duration_seconds: Histogram, + /// Time spent persisting a block. + pub block_write_duration_seconds: Histogram, + /// Number of account-trie updates written. + pub account_trie_updates_written_total: Counter, + /// Number of storage-trie updates written. + pub storage_trie_updates_written_total: Counter, + /// Number of account leaves written. + pub hashed_accounts_written_total: Counter, + /// Number of storage leaves written. + pub hashed_storages_written_total: Counter, + /// Earliest block currently served by proof history. + pub earliest_block: Gauge, + /// Latest block currently served by proof history. + pub latest_block: Gauge, + /// Number of prune failures. + pub prune_errors_total: Counter, + /// Number of unwind failures. + pub unwind_errors_total: Counter, +} + +impl ProofHistoryMetrics { + /// Records latency and write-count measurements for one processed block or batch. + pub fn record_block(durations: &OperationDurations, counts: Option<&WriteCounts>) { + let metrics = Self::default(); + metrics + .block_total_duration_seconds + .record(durations.total_duration_seconds.as_secs_f64()); + metrics + .block_execution_duration_seconds + .record(durations.execution_duration_seconds.as_secs_f64()); + metrics + .block_state_root_duration_seconds + .record(durations.state_root_duration_seconds.as_secs_f64()); + metrics + .block_write_duration_seconds + .record(durations.write_duration_seconds.as_secs_f64()); + if let Some(counts) = counts { + metrics + .account_trie_updates_written_total + .increment(counts.account_trie_updates_written_total); + metrics + .storage_trie_updates_written_total + .increment(counts.storage_trie_updates_written_total); + metrics + .hashed_accounts_written_total + .increment(counts.hashed_accounts_written_total); + metrics + .hashed_storages_written_total + .increment(counts.hashed_storages_written_total); + } + } + + /// Updates the gauges for the inclusive durable proof window. + pub fn set_window(earliest: u64, latest: u64) { + let metrics = Self::default(); + metrics.earliest_block.set(earliest as f64); + metrics.latest_block.set(latest as f64); + } + + pub(crate) fn record_prune_error() { + Self::default().prune_errors_total.increment(1); + } + + pub(crate) fn record_unwind_error() { + Self::default().unwind_errors_total.increment(1); + } +} diff --git a/crates/proofs/src/proof.rs b/crates/proofs/src/proof.rs new file mode 100644 index 0000000..5a1adeb --- /dev/null +++ b/crates/proofs/src/proof.rs @@ -0,0 +1,667 @@ +//! Provides proof operation implementations for [`MorphProofsStorage`]. + +use alloy_primitives::{ + Address, B256, Bytes, keccak256, + map::{B256Map, HashMap}, +}; +use reth_db::DatabaseError; +use reth_execution_errors::{StateProofError, StateRootError, StorageRootError, TrieWitnessError}; +use reth_trie::{ + StateRoot, StorageRoot, TrieType, + hashed_cursor::HashedPostStateCursorFactory, + metrics::TrieRootMetrics, + proof::{self, Proof}, + trie_cursor::InMemoryTrieCursorFactory, + witness::TrieWitness, +}; +use reth_trie_common::{ + AccountProof, ExecutionWitnessMode, HashedPostState, HashedPostStateSorted, HashedStorage, + MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::TrieUpdates, +}; + +use crate::{ + MorphProofsHashedAccountCursorFactory, MorphProofsStorage, MorphProofsStore, + MorphProofsTrieCursorFactory, +}; + +/// Build the trie + hashed cursor factories sharing one read transaction at the given block. +const fn from_tx<'tx, 'db, S>( + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, +) -> ( + MorphProofsTrieCursorFactory<'tx, 'db, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'db, S>, +) +where + S: MorphProofsStore + 'db, + 'db: 'tx, +{ + ( + MorphProofsTrieCursorFactory::new(storage, tx, block_number), + MorphProofsHashedAccountCursorFactory::new(storage, tx, block_number), + ) +} + +/// Extends [`Proof`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseProof<'tx, S: MorphProofsStore + 'tx> { + /// Generates the state proof for target account based on [`TrieInput`]. + fn overlay_account_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result; + + /// Generates an account proof while reusing an existing read transaction. + fn overlay_account_proof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result + where + 'tx: 'cursor; + + /// Generates the state [`MultiProof`] for target hashed account and storage keys. + fn overlay_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result; + + /// Generates a state multiproof while reusing an existing read transaction. + fn overlay_multiproof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result + where + 'tx: 'cursor; +} + +impl<'tx, S> DatabaseProof<'tx, S> + for Proof< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_account_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result { + let tx = storage.ro_tx().map_err(Into::::into)?; + Self::overlay_account_proof_with_tx(storage, &tx, block_number, input, address, slots) + } + + fn overlay_account_proof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result + where + 'tx: 'cursor, + { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = from_tx(storage, tx, block_number); + Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .account_proof(address, slots) + } + + fn overlay_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result { + let tx = storage.ro_tx().map_err(Into::::into)?; + Self::overlay_multiproof_with_tx(storage, &tx, block_number, input, targets) + } + + fn overlay_multiproof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result + where + 'tx: 'cursor, + { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = from_tx(storage, tx, block_number); + Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .multiproof(targets) + } +} + +/// Extends [`StorageProof`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseStorageProof<'tx, S: MorphProofsStore + 'tx> { + /// Generates the storage proof for target slot based on [`TrieInput`]. + fn overlay_storage_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> Result; + + /// Generates the storage multiproof for target slots based on [`TrieInput`]. + fn overlay_storage_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slots: &[B256], + storage: HashedStorage, + ) -> Result; +} + +impl<'tx, S> DatabaseStorageProof<'tx, S> + for proof::StorageProof< + 'static, + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_storage_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> Result { + let hashed_address = keccak256(address); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_proof(slot) + } + + fn overlay_storage_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slots: &[B256], + hashed_storage: HashedStorage, + ) -> Result { + let hashed_address = keccak256(address); + let targets = slots.iter().map(keccak256).collect(); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_multiproof(targets) + } +} + +/// Extends [`StateRoot`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseStateRoot<'tx, S: MorphProofsStore + 'tx + Clone>: Sized { + /// Calculate the state root for this [`HashedPostState`]. + /// Internally, this method retrieves prefixsets and uses them + /// to calculate incremental state root. + /// + /// # Returns + /// + /// The state root for this [`HashedPostState`]. + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result; + + /// Calculates the state root for this [`HashedPostState`] and returns it alongside trie + /// updates. See [`Self::overlay_root`] for more info. + fn overlay_root_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result<(B256, TrieUpdates), StateRootError>; + + /// Calculates the state root for provided [`HashedPostState`] using cached intermediate nodes. + fn overlay_root_from_nodes( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result; + + /// Calculates the state root and trie updates for provided [`HashedPostState`] using + /// cached intermediate nodes. + fn overlay_root_from_nodes_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result<(B256, TrieUpdates), StateRootError>; +} + +impl<'tx, S> DatabaseStateRoot<'tx, S> + for StateRoot< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result { + let prefix_sets = post_state.construct_prefix_sets().freeze(); + let state_sorted = post_state.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root() + } + + fn overlay_root_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result<(B256, TrieUpdates), StateRootError> { + let prefix_sets = post_state.construct_prefix_sets().freeze(); + let state_sorted = post_state.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root_with_updates() + } + + fn overlay_root_from_nodes( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root() + } + + fn overlay_root_from_nodes_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result<(B256, TrieUpdates), StateRootError> { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root_with_updates() + } +} + +/// Extends [`StorageRoot`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseStorageRoot<'tx, S: MorphProofsStore + 'tx + Clone> { + /// Calculates the storage root for provided [`HashedStorage`]. + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + hashed_storage: HashedStorage, + ) -> Result; +} + +impl<'tx, S> DatabaseStorageRoot<'tx, S> + for StorageRoot< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + hashed_storage: HashedStorage, + ) -> Result { + let prefix_set = hashed_storage.construct_prefix_set().freeze(); + let state_sorted = + HashedPostState::from_hashed_storage(keccak256(address), hashed_storage).into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StorageRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + address, + prefix_set, + TrieRootMetrics::new(TrieType::Custom("morph_historical_proofs_storage")), + ) + .root() + } +} + +/// Extends [`TrieWitness`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseTrieWitness<'tx, S: MorphProofsStore + 'tx + Clone> { + /// Generates the trie witness for the target state based on [`TrieInput`]. + fn overlay_witness( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError>; + + /// Generates the trie witness for the target state, reusing `tx`. + fn overlay_witness_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError> + where + 'tx: 'cursor; +} + +impl<'tx, S> DatabaseTrieWitness<'tx, S> + for TrieWitness< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_witness( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError> { + let tx = storage.ro_tx().map_err(|error| { + let error = Into::::into(error); + StateProofError::from(error) + })?; + Self::overlay_witness_with_tx(storage, &tx, block_number, input, target, mode) + } + + fn overlay_witness_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError> + where + 'tx: 'cursor, + { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = from_tx(storage, tx, block_number); + TrieWitness::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .always_include_root_node() + .with_execution_witness_mode(mode) + .compute(target) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; + use alloy_primitives::{Address, B256, U256, keccak256}; + use reth_primitives_traits::Account; + use reth_trie::EMPTY_ROOT_HASH; + + use super::*; + use crate::{ + BlockStateDiff, InMemoryProofsStorage, MdbxProofsStorage, MorphProofsInitialStateStore, + MorphProofsStore, + }; + + fn assert_account_and_storage_proof(storage: S) + where + S: MorphProofsStore + MorphProofsInitialStateStore + Clone, + { + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .unwrap(); + storage.commit_initial_state().unwrap(); + + let address = Address::repeat_byte(0x11); + let slot = B256::repeat_byte(0x22); + let value = U256::from(42); + let account = Account { + nonce: 7, + balance: U256::from(1_000_000), + bytecode_hash: None, + }; + let hashed_address = keccak256(address); + let post_state = HashedPostState::default() + .with_accounts([(hashed_address, Some(account))]) + .with_storages([( + hashed_address, + HashedStorage::from_iter(false, [(keccak256(slot), value)]), + )]); + + let (state_root, trie_updates) = + StateRoot::overlay_root_with_updates(&storage, 0, post_state.clone()).unwrap(); + let block_hash = B256::repeat_byte(0x02); + storage + .store_trie_updates( + BlockWithParent::new(genesis_hash, NumHash::new(1, block_hash)), + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }, + ) + .unwrap(); + + let proof = + Proof::overlay_account_proof(&storage, 1, TrieInput::default(), address, &[slot]) + .unwrap(); + + assert_eq!(proof.info, Some(account)); + assert_eq!(proof.storage_proofs.len(), 1); + assert_eq!(proof.storage_proofs[0].value, value); + proof.verify(state_root).unwrap(); + } + + #[test] + fn account_and_storage_proof_verify_in_memory() { + assert_account_and_storage_proof(InMemoryProofsStorage::new()); + } + + #[test] + fn account_and_storage_proof_verify_in_mdbx() { + let directory = tempfile::tempdir().unwrap(); + let storage = Arc::new(MdbxProofsStorage::new(directory.path()).unwrap()); + assert_account_and_storage_proof(storage); + } + + #[test] + fn proof_snapshot_survives_concurrent_prune_boundary_move() { + let directory = tempfile::tempdir().unwrap(); + let storage = Arc::new(MdbxProofsStorage::new(directory.path()).unwrap()); + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .unwrap(); + storage.commit_initial_state().unwrap(); + + let address = Address::repeat_byte(0x11); + let post_state = HashedPostState::default().with_accounts([( + keccak256(address), + Some(Account { + nonce: 1, + balance: U256::from(1), + bytecode_hash: None, + }), + )]); + let (_, trie_updates) = + StateRoot::overlay_root_with_updates(&storage, 0, post_state.clone()).unwrap(); + let block_one = + BlockWithParent::new(genesis_hash, NumHash::new(1, B256::repeat_byte(0x02))); + storage + .store_trie_updates( + block_one, + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }, + ) + .unwrap(); + + let request_tx = storage.ro_tx().unwrap(); + assert_eq!( + storage + .get_earliest_block_number_with_tx(&request_tx) + .unwrap() + .unwrap() + .0, + 0 + ); + + storage.prune_earliest_state(block_one).unwrap(); + assert_eq!(storage.get_earliest_block_number().unwrap().unwrap().0, 1); + assert_eq!( + storage + .get_earliest_block_number_with_tx(&request_tx) + .unwrap() + .unwrap() + .0, + 0, + "request snapshot must retain the validated proof window" + ); + + let proof = Proof::overlay_account_proof_with_tx( + &storage, + &request_tx, + 0, + TrieInput::default(), + address, + &[], + ) + .unwrap(); + assert!(proof.info.is_none()); + proof.verify(EMPTY_ROOT_HASH).unwrap(); + } + + #[test] + fn exclusion_proofs_verify_at_historical_block() { + let storage = InMemoryProofsStorage::new(); + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .unwrap(); + storage.commit_initial_state().unwrap(); + + let address = Address::repeat_byte(0x11); + let post_state = HashedPostState::default().with_accounts([( + keccak256(address), + Some(Account { + nonce: 1, + balance: U256::from(1), + bytecode_hash: None, + }), + )]); + let (state_root, trie_updates) = + StateRoot::overlay_root_with_updates(&storage, 0, post_state.clone()).unwrap(); + storage + .store_trie_updates( + BlockWithParent::new(genesis_hash, NumHash::new(1, B256::repeat_byte(0x02))), + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }, + ) + .unwrap(); + + let missing_address = Address::repeat_byte(0x33); + let missing_slot = B256::repeat_byte(0x44); + let proof = Proof::overlay_account_proof( + &storage, + 1, + TrieInput::default(), + missing_address, + &[missing_slot], + ) + .unwrap(); + + assert!(proof.info.is_none()); + assert_eq!(proof.storage_proofs.len(), 1); + assert_eq!(proof.storage_proofs[0].value, U256::ZERO); + proof.verify(state_root).unwrap(); + } +} diff --git a/crates/proofs/src/provider.rs b/crates/proofs/src/provider.rs new file mode 100644 index 0000000..2d9e292 --- /dev/null +++ b/crates/proofs/src/provider.rs @@ -0,0 +1,340 @@ +//! Provider for external proofs storage + +use std::fmt::Debug; + +use alloy_primitives::keccak256; +use parking_lot::{MappedMutexGuard, Mutex, MutexGuard}; +use reth_primitives_traits::{Account, Bytecode}; +use reth_provider::{ + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, ProviderError, + ProviderResult, StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, +}; +use reth_revm::{ + db::BundleState, + primitives::{Address, B256, Bytes, StorageValue, alloy_primitives::BlockNumber}, +}; +use reth_trie::{ + StateRoot, StorageRoot, + hashed_cursor::HashedCursor, + proof::{self, Proof}, + witness::TrieWitness, +}; +use reth_trie_common::{ + AccountProof, ExecutionWitnessMode, HashedPostState, HashedStorage, KeccakKeyHasher, + MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::TrieUpdates, +}; + +use crate::{ + MorphProofsStorage, MorphProofsStorageError, MorphProofsStore, + proof::{ + DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, + DatabaseTrieWitness, + }, +}; + +/// State provider for external proofs storage. +pub struct MorphProofsStateProviderRef<'a, Storage: MorphProofsStore> { + /// Historical state provider for non-state related tasks. + latest: Box, + + /// Storage provider for state lookups. + storage: &'a MorphProofsStorage, + + /// Max block number that can be used for state lookups. + block_number: BlockNumber, + + /// Lazily-acquired read-only transaction shared across all EVM state reads. + /// + /// Acquired once on the first [`basic_account`](AccountReader::basic_account) or + /// [`storage`](StateProvider::storage) call and reused for the lifetime of this provider, + /// so that all EVM state reads within a single execution context share one database snapshot + /// and avoid per-call transaction-acquisition contention. + lazy_tx: Mutex>>, +} + +impl<'a, Storage: MorphProofsStore> MorphProofsStateProviderRef<'a, Storage> { + /// Creates a new state provider. + pub fn new( + latest: Box, + storage: &'a MorphProofsStorage, + block_number: BlockNumber, + ) -> Self { + Self { + latest, + storage, + block_number, + lazy_tx: Mutex::new(None), + } + } + + /// Creates a state provider pinned to an existing proof-database read transaction. + pub fn new_with_tx( + latest: Box, + storage: &'a MorphProofsStorage, + block_number: BlockNumber, + tx: Storage::Tx<'a>, + ) -> Self { + Self { + latest, + storage, + block_number, + lazy_tx: Mutex::new(Some(tx)), + } + } + + fn ensure_tx(&self) -> ProviderResult>> { + let mut guard = self.lazy_tx.lock(); + if guard.is_none() { + *guard = Some(self.storage.ro_tx().map_err(Into::::into)?); + } + + Ok(MutexGuard::map(guard, |tx| { + tx.as_mut().expect("read-only transaction initialized") + })) + } +} + +impl<'a, Storage> Debug for MorphProofsStateProviderRef<'a, Storage> +where + Storage: MorphProofsStore + 'a + Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MorphProofsStateProviderRef") + .field("storage", &self.storage) + .field("block_number", &self.block_number) + .finish() + } +} + +impl<'a, Storage: MorphProofsStore + Clone> MorphProofsStateProviderRef<'a, Storage> { + fn storage_by_hashed_key( + &self, + address: Address, + hashed_key: B256, + ) -> ProviderResult> { + let tx = self.ensure_tx()?; + Ok(self + .storage + .storage_hashed_cursor_with_tx(&tx, keccak256(address.0), self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, val)| (key == hashed_key).then_some(val))) + } +} + +impl From for ProviderError { + fn from(error: MorphProofsStorageError) -> Self { + Self::other(error) + } +} + +impl<'a, Storage: MorphProofsStore> BlockHashReader for MorphProofsStateProviderRef<'a, Storage> { + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + self.latest.block_hash(number) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + self.latest.canonical_hashes_range(start, end) + } +} + +impl<'a, Storage: MorphProofsStore + Clone> StateRootProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn state_root(&self, state: HashedPostState) -> ProviderResult { + Ok(StateRoot::overlay_root( + self.storage, + self.block_number, + state, + )?) + } + + fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { + Ok(StateRoot::overlay_root_from_nodes( + self.storage, + self.block_number, + input, + )?) + } + + fn state_root_with_updates( + &self, + state: HashedPostState, + ) -> ProviderResult<(B256, TrieUpdates)> { + Ok(StateRoot::overlay_root_with_updates( + self.storage, + self.block_number, + state, + )?) + } + + fn state_root_from_nodes_with_updates( + &self, + input: TrieInput, + ) -> ProviderResult<(B256, TrieUpdates)> { + Ok(StateRoot::overlay_root_from_nodes_with_updates( + self.storage, + self.block_number, + input, + )?) + } +} + +impl<'a, Storage: MorphProofsStore + Clone> StorageRootProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + StorageRoot::overlay_root(self.storage, self.block_number, address, storage) + .map_err(|err| ProviderError::Database(err.into())) + } + + fn storage_proof( + &self, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> ProviderResult { + proof::StorageProof::overlay_storage_proof( + self.storage, + self.block_number, + address, + slot, + storage, + ) + .map_err(ProviderError::from) + } + + fn storage_multiproof( + &self, + address: Address, + slots: &[B256], + storage: HashedStorage, + ) -> ProviderResult { + proof::StorageProof::overlay_storage_multiproof( + self.storage, + self.block_number, + address, + slots, + storage, + ) + .map_err(ProviderError::from) + } +} + +impl<'a, Storage: MorphProofsStore + Clone> StateProofProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn proof( + &self, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> ProviderResult { + let tx = self.ensure_tx()?; + Proof::overlay_account_proof_with_tx( + self.storage, + &tx, + self.block_number, + input, + address, + slots, + ) + .map_err(ProviderError::from) + } + + fn multiproof( + &self, + input: TrieInput, + targets: MultiProofTargets, + ) -> ProviderResult { + let tx = self.ensure_tx()?; + Proof::overlay_multiproof_with_tx(self.storage, &tx, self.block_number, input, targets) + .map_err(ProviderError::from) + } + + fn witness( + &self, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> ProviderResult> { + let tx = self.ensure_tx()?; + TrieWitness::overlay_witness_with_tx( + self.storage, + &tx, + self.block_number, + input, + target, + mode, + ) + .map_err(ProviderError::from) + .map(|hm| hm.into_values().collect()) + } +} + +impl<'a, Storage: MorphProofsStore> HashedPostStateProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { + HashedPostState::from_bundle_state::(bundle_state.state()) + } +} + +impl<'a, Storage: MorphProofsStore> AccountReader for MorphProofsStateProviderRef<'a, Storage> { + fn basic_account(&self, address: &Address) -> ProviderResult> { + let hashed_key = keccak256(address.0); + let tx = self.ensure_tx()?; + Ok(self + .storage + .account_hashed_cursor_with_tx(&tx, self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, account)| (key == hashed_key).then_some(account))) + } +} + +impl<'a, Storage> StateProvider for MorphProofsStateProviderRef<'a, Storage> +where + Storage: MorphProofsStore + Clone, +{ + fn storage(&self, address: Address, storage_key: B256) -> ProviderResult> { + let hashed_key = keccak256(storage_key); + self.storage_by_hashed_key(address, hashed_key) + } +} + +impl<'a, Storage: MorphProofsStore> BytecodeReader for MorphProofsStateProviderRef<'a, Storage> { + fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { + self.latest.bytecode_by_hash(code_hash) + } +} + +#[cfg(test)] +mod tests { + use reth_provider::noop::NoopProvider; + + use super::*; + use crate::InMemoryProofsStorage; + + #[test] + fn test_morph_proofs_state_provider_ref_debug() { + let latest: Box = Box::::default(); + let storage: crate::MorphProofsStorage = + InMemoryProofsStorage::new(); + let block_number = 42u64; + + let provider = MorphProofsStateProviderRef::new(latest, &storage, block_number); + + assert_eq!( + format!("{provider:?}"), + "MorphProofsStateProviderRef { storage: InMemoryProofsStorage { inner: RwLock { data: InMemoryStorageInner { account_branches: {}, storage_branches: {}, hashed_accounts: {}, hashed_storages: {}, trie_updates: {}, post_states: {}, earliest_block: None, anchor_block: None } } }, block_number: 42 }" + ); + } +} diff --git a/crates/proofs/src/prune/error.rs b/crates/proofs/src/prune/error.rs new file mode 100644 index 0000000..56992c9 --- /dev/null +++ b/crates/proofs/src/prune/error.rs @@ -0,0 +1,102 @@ +use std::{ + fmt, + fmt::{Display, Formatter}, + time::Duration, +}; + +use reth_provider::ProviderError; +use strum::Display; +use thiserror::Error; + +use crate::{MorphProofsStorageError, api::WriteCounts}; + +/// Result of [`MorphProofStoragePruner::run`](crate::MorphProofStoragePruner::run) execution. +pub type MorphProofStoragePrunerResult = Result; + +/// Successful prune summary. +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct PrunerOutput { + /// Total elapsed wall time for this run (fetch + apply). + pub duration: Duration, + /// Earliest block at the start of the run. + pub start_block: u64, + /// New earliest block at the end of the run. + pub end_block: u64, + /// Number of entries updated/removed per table. + pub write_counts: WriteCounts, +} + +impl Display for PrunerOutput { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let blocks = self.end_block.saturating_sub(self.start_block); + let total_entries = self.write_counts.hashed_accounts_written_total + + self.write_counts.hashed_storages_written_total + + self.write_counts.account_trie_updates_written_total + + self.write_counts.storage_trie_updates_written_total; + write!( + f, + "Pruned {}→{} ({} blocks), entries={}, elapsed={:.3}s", + self.start_block, + self.end_block, + blocks, + total_entries, + self.duration.as_secs_f64(), + ) + } +} + +impl PrunerOutput { + /// extend the current [`PrunerOutput`] with another [`PrunerOutput`] + pub fn extend_ref(&mut self, other: Self) { + self.duration += other.duration; + // take the earliest start block + if self.start_block > other.start_block { + self.start_block = other.start_block; + } + // take the latest end block + if self.end_block < other.end_block { + self.end_block = other.end_block; + } + self.write_counts += other.write_counts; + } +} + +/// Error returned by the pruner. +#[derive(Debug, Error, Display)] +pub enum PrunerError { + /// Wrapped error from the underlying `MorphProofStorage` layer. + Storage(#[from] MorphProofsStorageError), + + /// Wrapped error from the reth db provider. + Provider(#[from] ProviderError), + + /// Block not found in the underlying reth storage provider. + BlockNotFound(u64), + + /// The pruner timed out before finishing the prune + TimedOut(Duration), +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::PrunerOutput; + use crate::api::WriteCounts; + + #[test] + fn test_pruner_output_display() { + let pruner_output = PrunerOutput { + duration: Duration::from_secs(10), + start_block: 1, + end_block: 2, + write_counts: WriteCounts::new(1, 2, 3, 4), + }; + let formatted_pruner_output = format!("{pruner_output}"); + + assert_eq!( + formatted_pruner_output, + "Pruned 1→2 (1 blocks), entries=10, elapsed=10.000s" + ); + } +} diff --git a/crates/proofs/src/prune/metrics.rs b/crates/proofs/src/prune/metrics.rs new file mode 100644 index 0000000..2ddc2a4 --- /dev/null +++ b/crates/proofs/src/prune/metrics.rs @@ -0,0 +1,54 @@ +//! Pruner metrics. + +use reth_metrics::{ + Metrics, + metrics::{Gauge, Histogram}, +}; + +use crate::PrunerOutput; + +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.proofs.pruner")] +pub(super) struct PrunerMetrics { + /// Pruning duration. + total_duration_seconds: Histogram, + /// Number of pruned blocks. + pruned_blocks: Gauge, + /// Number of account trie updates written. + account_trie_updates_written: Gauge, + /// Number of storage trie updates written. + storage_trie_updates_written: Gauge, + /// Number of hashed accounts written. + hashed_accounts_written: Gauge, + /// Number of hashed storages written. + hashed_storages_written: Gauge, +} + +impl PrunerMetrics { + pub(super) fn record_prune_result(result: PrunerOutput) { + let blocks_pruned = result.end_block.saturating_sub(result.start_block); + if blocks_pruned == 0 { + return; + } + + let metrics = Self::default(); + metrics + .total_duration_seconds + .record(result.duration.as_secs_f64()); + metrics.pruned_blocks.set(blocks_pruned as f64); + + let counts = &result.write_counts; + metrics + .account_trie_updates_written + .set(counts.account_trie_updates_written_total as f64); + metrics + .storage_trie_updates_written + .set(counts.storage_trie_updates_written_total as f64); + metrics + .hashed_accounts_written + .set(counts.hashed_accounts_written_total as f64); + metrics + .hashed_storages_written + .set(counts.hashed_storages_written_total as f64); + } +} diff --git a/crates/proofs/src/prune/mod.rs b/crates/proofs/src/prune/mod.rs new file mode 100644 index 0000000..4a5781a --- /dev/null +++ b/crates/proofs/src/prune/mod.rs @@ -0,0 +1,12 @@ +//! Proof storage pruner for removing stale trie data. + +mod error; +pub use error::{MorphProofStoragePrunerResult, PrunerError, PrunerOutput}; + +mod pruner; +pub use pruner::MorphProofStoragePruner; + +mod metrics; + +mod task; +pub use task::MorphProofStoragePrunerTask; diff --git a/crates/proofs/src/prune/pruner.rs b/crates/proofs/src/prune/pruner.rs new file mode 100644 index 0000000..4f64a78 --- /dev/null +++ b/crates/proofs/src/prune/pruner.rs @@ -0,0 +1,735 @@ +use std::cmp; + +use alloy_eips::{BlockNumHash, eip1898::BlockWithParent}; +use reth_provider::BlockHashReader; +use tokio::time::Instant; +use tracing::{error, info, trace}; + +use crate::{ + MorphProofsStorage, MorphProofsStore, ProofHistoryMetrics, + prune::{ + error::{MorphProofStoragePrunerResult, PrunerError, PrunerOutput}, + metrics::PrunerMetrics, + }, +}; + +/// Prunes the proof storage by calling `prune_earliest_state` on the storage provider. +#[derive(Debug)] +pub struct MorphProofStoragePruner { + /// Database provider for the prune + provider: MorphProofsStorage

, + /// Reader to fetch block hash by block number + block_hash_reader: H, + /// Keep at least these many recent blocks. + retention_blocks: u64, + /// Maximum number of blocks to prune in one database transaction + prune_batch_size: u64, +} + +impl MorphProofStoragePruner { + /// Create a new pruner. + pub const fn new( + provider: MorphProofsStorage

, + block_hash_reader: H, + retention_blocks: u64, + prune_batch_size: u64, + ) -> Self { + assert!( + prune_batch_size > 0, + "prune batch size must be greater than zero" + ); + Self { + provider, + block_hash_reader, + retention_blocks, + prune_batch_size, + } + } + + const fn target_earliest_block(&self, latest_block: u64) -> u64 { + latest_block.saturating_sub(self.retention_blocks) + } +} + +impl MorphProofStoragePruner +where + P: MorphProofsStore, + H: BlockHashReader, +{ + fn run_inner(&self) -> MorphProofStoragePrunerResult { + let Some(((earliest_block, _), (latest_block, _))) = self + .provider + .get_earliest_block_number()? + .zip(self.provider.get_latest_block_number()?) + else { + trace!(target: "trie::pruner", "No earliest or latest block in the proof storage"); + return Ok(PrunerOutput::default()); + }; + + let target_earliest_block = self.target_earliest_block(latest_block); + info!( + target: "trie::pruner", + earliest_block, + latest_block, + target_earliest_block, + retention_blocks = self.retention_blocks, + prune_batch_size = self.prune_batch_size, + "Calculated proof storage pruning target", + ); + if earliest_block >= target_earliest_block { + trace!(target: "trie::pruner", "Nothing to prune"); + return Ok(PrunerOutput::default()); + } + + info!( + target: "trie::pruner", + from_block = earliest_block, + to_block = target_earliest_block, + latest_block, + retention_blocks = self.retention_blocks, + "Starting pruning proof storage", + ); + + let mut current_earliest_block = earliest_block; + let mut prune_output = PrunerOutput { + start_block: earliest_block, + end_block: target_earliest_block, + ..Default::default() + }; + + // Prune in batches + while current_earliest_block < target_earliest_block { + // Calculate the end of this batch + let batch_end_block = cmp::min( + current_earliest_block.saturating_add(self.prune_batch_size), + target_earliest_block, + ); + info!( + target: "trie::pruner", + start_block = current_earliest_block, + end_block = batch_end_block, + target_earliest_block, + batch_size = batch_end_block.saturating_sub(current_earliest_block), + "Starting proof storage prune batch", + ); + + let batch_output = self.prune_batch(current_earliest_block, batch_end_block)?; + + prune_output.extend_ref(batch_output); + + // Update loop state + current_earliest_block = batch_end_block; + } + + Ok(prune_output) + } + + /// Prunes a single batch of blocks. + fn prune_batch(&self, start_block: u64, end_block: u64) -> Result { + let batch_start_time = Instant::now(); + info!( + target: "trie::pruner", + start_block, + end_block, + "Resolving proof storage prune batch block hashes", + ); + if end_block == 0 { + trace!( + target: "trie::pruner", + start_block, + end_block, + "Skipping proof storage prune batch at genesis block", + ); + return Ok(PrunerOutput { + duration: batch_start_time.elapsed(), + start_block, + end_block, + ..Default::default() + }); + } + + // Fetch the block hash for the new earliest block of this batch. + // + // The parent hash is intentionally not fetched: `prune_earliest_state` only reads + // `block.number` and `block.hash` from the supplied `BlockWithParent`. Fetching the + // parent hash here was wasted I/O proportional to the number of batches. + let new_earliest_block_hash = self + .block_hash_reader + .block_hash(end_block) + .inspect_err(|err| { + error!( + target: "trie::pruner", + block = end_block, + ?err, + "Failed to fetch block hash for new earliest block during pruning" + ) + })? + .ok_or(PrunerError::BlockNotFound(end_block))?; + + let block_with_parent = BlockWithParent { + parent: Default::default(), + block: BlockNumHash { + number: end_block, + hash: new_earliest_block_hash, + }, + }; + + info!( + target: "trie::pruner", + start_block, + end_block, + block_hash = ?new_earliest_block_hash, + "Resolved proof storage prune batch block hashes", + ); + + info!( + target: "trie::pruner", + start_block, + end_block, + "Applying proof storage prune batch", + ); + let write_counts = self.provider.prune_earliest_state(block_with_parent)?; + + let duration = batch_start_time.elapsed(); + let batch_output = PrunerOutput { + duration, + start_block, + end_block, + write_counts, + }; + + PrunerMetrics::record_prune_result(batch_output.clone()); + + info!( + target: "trie::pruner", + ?batch_output, + "Finished pruning batch of proof storage", + ); + Ok(batch_output) + } + + /// Run the pruner + pub fn run(&self) { + let res = self.try_run(); + match res { + Err(e) => { + error!(target: "trie::pruner", err=%e, "Pruner failed"); + } + Ok(res) => { + info!(target: "trie::pruner", result = %res, "Finished pruning proof storage"); + } + } + } + + /// Runs the pruner and returns failures to command-style callers. + pub fn try_run(&self) -> MorphProofStoragePrunerResult { + let result = self.run_inner(); + match &result { + Ok(_) => { + if let Ok(Some((earliest, _))) = self.provider.get_earliest_block_number() + && let Ok(Some((latest, _))) = self.provider.get_latest_block_number() + { + ProofHistoryMetrics::set_window(earliest, latest); + } + } + Err(_) => ProofHistoryMetrics::record_prune_error(), + } + result + } +} + +#[cfg(test)] +mod tests { + use alloy_eips::{BlockHashOrNumber, NumHash}; + use alloy_primitives::{B256, BlockNumber, keccak256}; + use mockall::mock; + use reth_storage_errors::provider::ProviderResult; + + use super::*; + + mock! ( + #[derive(Debug)] + pub BlockHashReader {} + + impl BlockHashReader for BlockHashReader { + fn block_hash(&self, number: BlockNumber) -> ProviderResult>; + + fn convert_block_hash( + &self, + _hash_or_number: BlockHashOrNumber, + ) -> ProviderResult>; + + fn canonical_hashes_range( + &self, + _start: BlockNumber, + _end: BlockNumber, + ) -> ProviderResult>; + } + ); + + fn b256(n: u64) -> B256 { + keccak256(n.to_be_bytes()) + } + + /// Build a block-with-parent for number `n` with deterministic hash. + fn block(n: u64, parent: B256) -> BlockWithParent { + BlockWithParent::new(parent, NumHash::new(n, b256(n))) + } + + macro_rules! proof_storage_pruner_tests { + ($mod_name:ident, $storage:ident) => { + mod $mod_name { + use std::sync::Arc; + + use alloy_primitives::{B256, U256}; + use reth_primitives_traits::Account; + use reth_trie::{ + BranchNodeCompact, HashedPostState, HashedStorage, Nibbles, + hashed_cursor::HashedCursor, + trie_cursor::TrieCursor, + updates::{StorageTrieUpdates, TrieUpdates, TrieUpdatesSorted}, + }; + use tempfile::TempDir; + + use super::{super::*, MockBlockHashReader, b256, block}; + use crate::{BlockStateDiff, $storage}; + + #[tokio::test] + async fn run_inner_and_and_verify_updated_state() { + // --- env/store --- + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + store + .set_earliest_block_number(0, B256::ZERO) + .expect("set earliest"); + + // --- entities --- + // accounts + let a1 = B256::from([0xA1; 32]); + let a2 = B256::from([0xA2; 32]); + let a3 = B256::from([0xA3; 32]); // introduced later + + // one storage address with 3 slots + let stor_addr = B256::from([0x10; 32]); + let s1 = B256::from([0xB1; 32]); + let s2 = B256::from([0xB2; 32]); + let s3 = B256::from([0xB3; 32]); + + // account-trie paths (p1 gets removed by block 3; p2 remains; p3 added later) + let p1 = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let p2 = Nibbles::from_nibbles_unchecked([0x03, 0x04]); + let p3 = Nibbles::from_nibbles_unchecked([0x05, 0x06]); + + let node_p1 = + BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::from([0x11; 32]))); + let node_p2 = + BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::from([0x22; 32]))); + let node_p3 = + BranchNodeCompact::new(0b11, 0, 0, vec![], Some(B256::from([0x33; 32]))); + + // storage-trie paths (st1 removed by block 3; st2 remains; st3 added later) + let st1 = Nibbles::from_nibbles_unchecked([0x0A]); + let st2 = Nibbles::from_nibbles_unchecked([0x0B]); + let st3 = Nibbles::from_nibbles_unchecked([0x0C]); + + let node_st2 = + BranchNodeCompact::new(0b101, 0, 0, vec![], Some(B256::from([0x44; 32]))); + let node_st3 = + BranchNodeCompact::new(0b110, 0, 0, vec![], Some(B256::from([0x55; 32]))); + + // --- write 5 blocks manually --- + let mut parent = B256::ZERO; + + // Block 1: add a1,a2; s1=100, s2=200; add p1, st1 + { + let b1 = block(1, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a1, + Some(Account { + nonce: 1, + balance: U256::from(1_001), + ..Default::default() + }), + ); + d_post_state.accounts.insert( + a2, + Some(Account { + nonce: 1, + balance: U256::from(1_002), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s1, U256::from(100)); + hs.storage.insert(s2, U256::from(200)); + d_post_state.storages.insert(stor_addr, hs); + + d_trie_updates.account_nodes.insert(p1, node_p1); + let e = d_trie_updates.storage_tries.entry(stor_addr).or_default(); + e.storage_nodes.insert(st1, BranchNodeCompact::default()); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b1, d).expect("b1"); + parent = b256(1); + } + + // Block 2: update a2; add a3; s2=220, s3=300; add p2, st2 + { + let b2 = block(2, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a2, + Some(Account { + nonce: 2, + balance: U256::from(2_002), + ..Default::default() + }), + ); + d_post_state.accounts.insert( + a3, + Some(Account { + nonce: 1, + balance: U256::from(1_003), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s2, U256::from(220)); + hs.storage.insert(s3, U256::from(300)); + d_post_state.storages.insert(stor_addr, hs); + + d_trie_updates.account_nodes.insert(p2, node_p2.clone()); + let e = d_trie_updates.storage_tries.entry(stor_addr).or_default(); + e.storage_nodes.insert(st2, node_st2.clone()); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b2, d).expect("b2"); + parent = b256(2); + } + + // Block 3: delete a1; leave a2,a3; remove p1; remove st1 (storage-trie) + { + let b3 = block(3, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + // delete a1, keep a2 & a3 values unchanged for this block + d_post_state.accounts.insert(a1, None); + + // remove account trie node p1 + d_trie_updates.removed_nodes.insert(p1); + + // remove storage-trie node st1 + let mut st_upd = StorageTrieUpdates::default(); + st_upd.removed_nodes.insert(st1); + d_trie_updates.storage_tries.insert(stor_addr, st_upd); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b3, d).expect("b3"); + parent = b256(3); + } + + // Block 4 (kept): update a2; s1=140; add p3, st3 + { + let b4 = block(4, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a2, + Some(Account { + nonce: 3, + balance: U256::from(3_002), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s1, U256::from(140)); + d_post_state.storages.insert(stor_addr, hs); + d_trie_updates.account_nodes.insert(p3, node_p3.clone()); + let e = d_trie_updates.storage_tries.entry(stor_addr).or_default(); + e.storage_nodes.insert(st3, node_st3.clone()); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b4, d).expect("b4"); + parent = b256(4); + } + + // Block 5 (kept): update a3; s3=330 + { + let b5 = block(5, parent); + + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a3, + Some(Account { + nonce: 2, + balance: U256::from(2_003), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s3, U256::from(330)); + d_post_state.storages.insert(stor_addr, hs); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: TrieUpdatesSorted::default(), + }; + store.store_trie_updates(b5, d).expect("b5"); + } + + // sanity: earliest=0, latest=5 + { + let e = store + .get_earliest_block_number() + .expect("earliest") + .expect("some"); + let l = store + .get_latest_block_number() + .expect("latest") + .expect("some"); + assert_eq!(e.0, 0); + assert_eq!(l.0, 5); + } + + // --- prune: remove the first 3 blocks, keep 4 and 5 + // new_earliest = 5-1 = 4 + let mut block_hash_reader = MockBlockHashReader::new(); + block_hash_reader + .expect_block_hash() + .withf(move |block_num| *block_num == 4) + .returning(move |_| Ok(Some(b256(4)))); + + let pruner = + MorphProofStoragePruner::new(store.clone(), block_hash_reader, 1, 1000); + let out = pruner.run_inner().expect("pruner ok"); + assert_eq!(out.start_block, 0); + assert_eq!( + out.end_block, 4, + "pruned up to 4 (inclusive); new earliest is 4" + ); + + // proof window moved: earliest=4, latest=5 + { + let e = store + .get_earliest_block_number() + .expect("earliest") + .expect("some"); + let l = store + .get_latest_block_number() + .expect("latest") + .expect("some"); + assert_eq!(e.0, 4); + assert_eq!(e.1, b256(4)); + assert_eq!(l.0, 5); + assert_eq!(l.1, b256(5)); + } + + // --- DB checks + let mut acc_cur = store.account_hashed_cursor(4).expect("acc cur"); + let mut stor_cur = store.storage_hashed_cursor(stor_addr, 4).expect("stor cur"); + let mut acc_trie_cur = store.account_trie_cursor(4).expect("acc trie cur"); + let mut stor_trie_cur = store + .storage_trie_cursor(stor_addr, 4) + .expect("stor trie cur"); + + // Check these histories have been removed + let pruned_hashed_account = a1; + let pruned_trie_accounts = p1; + let pruned_trie_storage = st1; + + assert_ne!( + acc_cur + .seek(pruned_hashed_account) + .expect("seek") + .unwrap() + .0, + pruned_hashed_account, + "deleted account must not exist in earliest snapshot" + ); + assert_ne!( + acc_trie_cur + .seek(pruned_trie_accounts) + .expect("seek") + .unwrap() + .0, + pruned_trie_accounts, + "deleted account trie must not exist in earliest snapshot" + ); + assert_ne!( + stor_trie_cur + .seek(pruned_trie_storage) + .expect("seek") + .unwrap() + .0, + pruned_trie_storage, + "deleted storage trie must not exist in earliest snapshot" + ); + + // Check these histories have been updated - till block 4 + let updated_hashed_accounts = vec![ + ( + a2, + Account { + nonce: 3, + balance: U256::from(3_002), + ..Default::default() + }, + ), + ( + a3, + Account { + nonce: 1, + balance: U256::from(1_003), + ..Default::default() + }, + ), + ]; + let updated_hashed_storage = vec![ + (s1, U256::from(140)), + (s2, U256::from(220)), + (s3, U256::from(300)), + ]; + let updated_trie_accounts = vec![(p2, node_p2), (p3, node_p3)]; + let updated_trie_storage = vec![(st2, node_st2), (st3, node_st3)]; + + for (key, val) in updated_hashed_accounts { + let (k, vv) = acc_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + + for (key, val) in updated_hashed_storage { + let (k, vv) = stor_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + + for (key, val) in updated_trie_accounts { + let (k, vv) = acc_trie_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + for (key, val) in updated_trie_storage { + let (k, vv) = stor_trie_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + } + + // Both latest and earliest blocks are None -> early return default; DB untouched. + #[tokio::test] + async fn run_inner_where_latest_block_is_none() { + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + let earliest = store.get_earliest_block_number().unwrap(); + let latest = store.get_latest_block_number().unwrap(); + assert!(earliest.is_none()); + assert!(latest.is_none()); + + let block_hash_reader = MockBlockHashReader::new(); + let pruner = MorphProofStoragePruner::new(store, block_hash_reader, 10, 1000); + let out = pruner.run_inner().expect("ok"); + assert_eq!( + out, + PrunerOutput::default(), + "should early-return default output" + ); + } + + // The earliest block is None, but the latest block exists -> early return default. + #[tokio::test] + async fn run_inner_earliest_none_real_db() { + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + // Write a single block to set *latest* only. + store + .store_trie_updates(block(3, B256::ZERO), BlockStateDiff::default()) + .expect("store b1"); + + let earliest = store.get_earliest_block_number().unwrap(); + let latest = store.get_latest_block_number().unwrap(); + assert!(earliest.is_none(), "earliest must remain None"); + assert_eq!(latest.unwrap().0, 3); + + let block_hash_reader = MockBlockHashReader::new(); + let pruner = MorphProofStoragePruner::new(store, block_hash_reader, 1, 1000); + let out = pruner.run_inner().expect("ok"); + assert_eq!( + out, + PrunerOutput::default(), + "should early-return default output" + ); + } + + // interval < min_block_interval -> "Nothing to prune" path; default output. + #[tokio::test] + async fn run_inner_interval_too_small_real_db() { + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + // Set earliest=4 explicitly + let earliest_num = 4u64; + let h4 = b256(4); + store + .set_earliest_block_number(earliest_num, h4) + .expect("set earliest"); + + // Set latest=5 by storing block 5 + let b5 = block(5, h4); + store + .store_trie_updates(b5, BlockStateDiff::default()) + .expect("store b5"); + + // Sanity: earliest=4, latest=5 => interval=1 + let e = store.get_earliest_block_number().unwrap().unwrap(); + let l = store.get_latest_block_number().unwrap().unwrap(); + assert_eq!(e.0, 4); + assert_eq!(l.0, 5); + + // Require min_block_interval=2 (or greater) so interval < min + let block_hash_reader = MockBlockHashReader::new(); + let pruner = MorphProofStoragePruner::new(store, block_hash_reader, 2, 1000); + let out = pruner.run_inner().expect("ok"); + assert_eq!(out, PrunerOutput::default(), "no pruning should occur"); + } + } + }; + } + + proof_storage_pruner_tests!(mdbx_tests, MdbxProofsStorage); +} diff --git a/crates/proofs/src/prune/task.rs b/crates/proofs/src/prune/task.rs new file mode 100644 index 0000000..d9f4fb8 --- /dev/null +++ b/crates/proofs/src/prune/task.rs @@ -0,0 +1,80 @@ +use std::sync::Arc; + +use reth_provider::BlockHashReader; +use reth_tasks::shutdown::GracefulShutdown; +use tokio::{ + time, + time::{Duration, MissedTickBehavior}, +}; +use tracing::{info, warn}; + +use crate::{MorphProofsStorage, MorphProofsStore, prune::MorphProofStoragePruner}; + +/// Number of blocks pruned per MDBX write transaction. +/// +/// Each batch is its own write tx, so the per-tx overhead (page allocation, freelist mgmt, +/// fsync at commit) amortizes over this many blocks. The previous value of 200 caused tx +/// overhead to dominate catch-up pruning runs; 2000 amortizes the fixed cost ~10x while +/// keeping per-tx dirty-page sets and free-page reclamation lag bounded. +const PRUNE_BATCH_SIZE: u64 = 2000; + +/// Periodic pruner task: constructs the pruner and runs it every interval. +#[derive(Debug)] +pub struct MorphProofStoragePrunerTask { + pruner: MorphProofStoragePruner, + retention_blocks: u64, + task_run_interval: Duration, +} + +impl MorphProofStoragePrunerTask +where + P: MorphProofsStore + Send + Sync + 'static, + H: BlockHashReader + Send + Sync + 'static, +{ + /// Initialize a new [`MorphProofStoragePrunerTask`] + pub const fn new( + provider: MorphProofsStorage

, + hash_reader: H, + retention_blocks: u64, + task_run_interval: Duration, + ) -> Self { + let pruner = + MorphProofStoragePruner::new(provider, hash_reader, retention_blocks, PRUNE_BATCH_SIZE); + Self { + pruner, + retention_blocks, + task_run_interval, + } + } + + /// Run forever (until `cancel`), executing one prune pass per `task_run_interval`. + pub async fn run(self, mut signal: GracefulShutdown) { + info!( + target: "trie::pruner_task", + retention_blocks = self.retention_blocks, + interval_secs = self.task_run_interval.as_secs(), + "Starting pruner task" + ); + + let mut interval = time::interval(self.task_run_interval); + interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + + let pruner = Arc::new(self.pruner); + loop { + tokio::select! { + _ = &mut signal => { + info!(target: "trie::pruner_task", "Pruner task cancelled; exiting"); + break; + } + _ = interval.tick() => { + let pruner = Arc::clone(&pruner); + if let Err(e) = tokio::task::spawn_blocking(move || pruner.run()).await { + warn!(target: "trie::pruner_task", err=%e, "Pruner blocking task failed"); + } + } + } + } + + info!(target: "trie::pruner_task", "Pruner task stopped"); + } +} diff --git a/crates/proofs/tests/identity.rs b/crates/proofs/tests/identity.rs new file mode 100644 index 0000000..c45cb71 --- /dev/null +++ b/crates/proofs/tests/identity.rs @@ -0,0 +1,24 @@ +use alloy_primitives::B256; +use morph_proofs::{MdbxProofsStorage, MorphProofsStorageError, ProofDbIdentity}; + +#[test] +fn opening_a_store_persists_and_validates_chain_identity() { + let directory = tempfile::tempdir().unwrap(); + let identity = ProofDbIdentity::new(2818, B256::repeat_byte(0x11)); + + let store = MdbxProofsStorage::open(directory.path(), identity).unwrap(); + assert_eq!(store.proof_window().unwrap(), None); + drop(store); + + MdbxProofsStorage::open(directory.path(), identity).unwrap(); + let error = MdbxProofsStorage::open( + directory.path(), + ProofDbIdentity::new(2819, identity.genesis_hash), + ) + .unwrap_err(); + + assert!(matches!( + error, + MorphProofsStorageError::ChainIdentityMismatch("chain_id") + )); +} diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 356a7cb..e7539dc 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -16,6 +16,7 @@ morph-chainspec.workspace = true morph-reference-index.workspace = true morph-revm = { workspace = true, features = ["rpc"] } morph-evm = { workspace = true, features = ["rpc"] } +morph-proofs.workspace = true # Reth reth-chainspec.workspace = true @@ -23,9 +24,13 @@ reth-node-api.workspace = true reth-node-builder.workspace = true reth-primitives-traits.workspace = true reth-rpc.workspace = true +reth-rpc-api.workspace = true reth-rpc-convert.workspace = true reth-rpc-eth-api.workspace = true reth-rpc-eth-types.workspace = true +reth-rpc-server-types.workspace = true +reth-metrics.workspace = true +metrics.workspace = true reth-evm.workspace = true reth-provider.workspace = true reth-revm.workspace = true @@ -45,6 +50,7 @@ alloy-eips.workspace = true # JSON-RPC jsonrpsee.workspace = true +async-trait.workspace = true # Serialization serde.workspace = true diff --git a/crates/rpc/src/eth/mod.rs b/crates/rpc/src/eth/mod.rs index cb7331a..9e325b6 100644 --- a/crates/rpc/src/eth/mod.rs +++ b/crates/rpc/src/eth/mod.rs @@ -32,6 +32,7 @@ use reth_tasks::{ use std::{fmt, marker::PhantomData, sync::Arc, time::Duration}; pub mod call; +pub mod proofs; pub mod receipt; pub mod transaction; diff --git a/crates/rpc/src/eth/proofs.rs b/crates/rpc/src/eth/proofs.rs new file mode 100644 index 0000000..f63c6ca --- /dev/null +++ b/crates/rpc/src/eth/proofs.rs @@ -0,0 +1,146 @@ +//! Historical EIP-1186 proof RPC override. + +use std::time::Instant; + +use alloy_eips::BlockId; +use alloy_primitives::Address; +use alloy_rpc_types_eth::EIP1186AccountProofResponse; +use alloy_serde::JsonStorageKey; +use async_trait::async_trait; +use jsonrpsee::{ + core::RpcResult, + proc_macros::rpc, + types::{ErrorCode, ErrorObject}, +}; +use morph_proofs::{MorphProofsStorage, MorphProofsStore}; +use reth_metrics::{ + Metrics, + metrics::{Counter, Histogram}, +}; +use reth_provider::StateProofProvider; +use reth_rpc_api::eth::helpers::FullEthApi; +use reth_rpc_eth_types::EthApiError; + +use crate::state::MorphProofStateProviderFactory; + +/// Maximum number of storage keys accepted by one `eth_getProof` request. +pub const MAX_PROOF_KEYS: usize = 1024; + +/// Validates the storage-key count before any database work starts. +#[derive(Debug)] +pub struct ProofKeyLimit; + +impl ProofKeyLimit { + /// Rejects requests above [`MAX_PROOF_KEYS`]. + pub fn check(keys_len: usize) -> Result<(), ErrorObject<'static>> { + if keys_len > MAX_PROOF_KEYS { + return Err(ErrorObject::owned( + ErrorCode::InvalidParams.code(), + format!("too many storage keys: max {MAX_PROOF_KEYS}, got {keys_len}"), + None::<()>, + )); + } + Ok(()) + } +} + +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.rpc.proofs")] +struct ProofRpcMetrics { + /// Total `eth_getProof` requests. + requests_total: Counter, + /// Successful `eth_getProof` responses. + successful_responses_total: Counter, + /// Failed `eth_getProof` responses. + failures_total: Counter, + /// Successful request latency in seconds. + latency_seconds: Histogram, +} + +#[cfg_attr(not(test), rpc(server, namespace = "eth"))] +#[cfg_attr(test, rpc(server, client, namespace = "eth"))] +pub trait EthProofApiOverride { + /// Returns EIP-1186 account and storage proofs at a retained canonical block. + #[method(name = "getProof")] + async fn get_proof( + &self, + address: Address, + keys: Vec, + block: Option, + ) -> RpcResult; +} + +/// `eth_getProof` implementation backed exclusively by Morph proof history. +#[derive(Debug)] +pub struct EthProofApiExt { + state_provider_factory: MorphProofStateProviderFactory, +} + +impl EthProofApiExt +where + Eth: FullEthApi + Send + Sync + 'static, + P: MorphProofsStore + Clone + 'static, +{ + /// Creates the historical proof RPC override. + pub const fn new(eth_api: Eth, storage: MorphProofsStorage

) -> Self { + Self { + state_provider_factory: MorphProofStateProviderFactory::new(eth_api, storage), + } + } +} + +#[async_trait] +impl EthProofApiOverrideServer for EthProofApiExt +where + Eth: FullEthApi + Send + Sync + 'static, + P: MorphProofsStore + Clone + 'static, +{ + async fn get_proof( + &self, + address: Address, + keys: Vec, + block: Option, + ) -> RpcResult { + ProofKeyLimit::check(keys.len())?; + + let metrics = ProofRpcMetrics::default(); + let start = Instant::now(); + metrics.requests_total.increment(1); + let storage_keys = keys.iter().map(JsonStorageKey::as_b256).collect::>(); + + let result = (|| { + let proof = self + .state_provider_factory + .state_provider(block) + .map_err(|error| ErrorObject::from(EthApiError::from(error)))? + .proof(Default::default(), address, &storage_keys) + .map_err(|error| ErrorObject::from(EthApiError::from(error)))?; + Ok(proof.into_eip1186_response(keys)) + })(); + + match &result { + Ok(_) => { + metrics + .latency_seconds + .record(start.elapsed().as_secs_f64()); + metrics.successful_responses_total.increment(1); + } + Err(_) => metrics.failures_total.increment(1), + } + result + } +} + +#[cfg(test)] +mod tests { + use jsonrpsee::types::ErrorCode; + + use super::{MAX_PROOF_KEYS, ProofKeyLimit}; + + #[test] + fn enforces_get_proof_key_limit() { + assert!(ProofKeyLimit::check(MAX_PROOF_KEYS).is_ok()); + let error = ProofKeyLimit::check(MAX_PROOF_KEYS + 1).expect_err("must reject"); + assert_eq!(error.code(), ErrorCode::InvalidParams.code()); + } +} diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index e7efe37..eff5e95 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -6,10 +6,13 @@ pub mod error; pub mod eth; pub mod eth_config; pub mod morph; +pub mod proof_status; +pub mod state; pub mod types; pub use error::MorphEthApiError; pub use eth::{MorphEthApi, MorphEthApiBuilder, MorphRpcConverter, MorphRpcTypes}; pub use eth_config::{MorphEthConfigApiServer, MorphEthConfigHandler}; pub use morph::{MorphRpc, MorphRpcHandler, MorphRpcServer, ReferenceQueryArgs}; +pub use proof_status::{ProofStatusApiExt, ProofStatusApiOverrideServer, ProofsSyncStatus}; pub use types::*; diff --git a/crates/rpc/src/proof_status.rs b/crates/rpc/src/proof_status.rs new file mode 100644 index 0000000..23424a6 --- /dev/null +++ b/crates/rpc/src/proof_status.rs @@ -0,0 +1,130 @@ +//! Proof-history synchronization status RPC. + +use async_trait::async_trait; +use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use morph_proofs::{MorphProofsStorage, MorphProofsStorageResult, MorphProofsStore}; +use reth_rpc_server_types::result::internal_rpc_err; +use serde::{Deserialize, Serialize}; + +/// Inclusive proof-history range currently available to RPC requests. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct ProofsSyncStatus { + /// Earliest retained canonical block. + pub earliest: Option, + /// Latest durably indexed canonical block. + pub latest: Option, +} + +#[cfg_attr(not(test), rpc(server, namespace = "debug"))] +#[cfg_attr(test, rpc(server, client, namespace = "debug"))] +pub trait ProofStatusApiOverride { + /// Returns the current durable proof-history range. + #[method(name = "proofsSyncStatus")] + async fn proofs_sync_status(&self) -> RpcResult; +} + +/// `debug_proofsSyncStatus` backed by the proof database. +#[derive(Debug)] +pub struct ProofStatusApiExt

{ + storage: MorphProofsStorage

, +} + +impl

ProofStatusApiExt

{ + /// Creates a proof-status handler. + pub const fn new(storage: MorphProofsStorage

) -> Self { + Self { storage } + } +} + +fn proofs_sync_status_with_tx<'tx, 'db, P>( + storage: &P, + tx: &'tx P::Tx<'db>, +) -> MorphProofsStorageResult +where + P: MorphProofsStore + 'db, + 'db: 'tx, +{ + let earliest = storage + .get_earliest_block_number_with_tx(tx)? + .map(|(number, _)| number); + let latest = storage + .get_latest_block_number_with_tx(tx)? + .map(|(number, _)| number); + Ok(ProofsSyncStatus { earliest, latest }) +} + +#[async_trait] +impl

ProofStatusApiOverrideServer for ProofStatusApiExt

+where + P: MorphProofsStore + Clone + Send + Sync + 'static, +{ + async fn proofs_sync_status(&self) -> RpcResult { + // Keep both bounds on the same MDBX snapshot. A concurrent prune may advance the live + // earliest block, but it must not produce a status range assembled from two generations. + let tx = self + .storage + .ro_tx() + .map_err(|error| internal_rpc_err(error.to_string()))?; + proofs_sync_status_with_tx(&self.storage, &tx) + .map_err(|error| internal_rpc_err(error.to_string())) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; + use alloy_primitives::B256; + use morph_proofs::{ + BlockStateDiff, MdbxProofsStorage, MorphProofsInitialStateStore, MorphProofsStore, + }; + + use super::{ProofsSyncStatus, proofs_sync_status_with_tx}; + + #[test] + fn empty_status_uses_json_nulls() { + let json = serde_json::to_value(ProofsSyncStatus { + earliest: None, + latest: None, + }) + .expect("serialize status"); + assert!(json["earliest"].is_null()); + assert!(json["latest"].is_null()); + } + + #[test] + fn status_bounds_share_one_snapshot_during_prune() { + let directory = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(MdbxProofsStorage::new(directory.path()).expect("open storage")); + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .expect("set initial anchor"); + storage + .commit_initial_state() + .expect("commit initial state"); + + let block_one = + BlockWithParent::new(genesis_hash, NumHash::new(1, B256::repeat_byte(0x02))); + storage + .store_trie_updates(block_one, BlockStateDiff::default()) + .expect("store block one"); + + let request_tx = storage.ro_tx().expect("open status snapshot"); + storage + .prune_earliest_state(block_one) + .expect("advance live proof window"); + + let request_status = proofs_sync_status_with_tx(&storage, &request_tx) + .expect("read status from pinned snapshot"); + assert_eq!(request_status.earliest, Some(0)); + assert_eq!(request_status.latest, Some(1)); + + let current_tx = storage.ro_tx().expect("open current snapshot"); + let current_status = + proofs_sync_status_with_tx(&storage, ¤t_tx).expect("read current status"); + assert_eq!(current_status.earliest, Some(1)); + assert_eq!(current_status.latest, Some(1)); + } +} diff --git a/crates/rpc/src/state.rs b/crates/rpc/src/state.rs new file mode 100644 index 0000000..657fc7d --- /dev/null +++ b/crates/rpc/src/state.rs @@ -0,0 +1,185 @@ +//! State-provider routing for bounded historical proofs. + +use alloy_eips::BlockId; +use alloy_primitives::B256; +use morph_proofs::{MorphProofsStorage, MorphProofsStore, provider::MorphProofsStateProviderRef}; +use reth_provider::{ + BlockHashReader, BlockIdReader, ProviderError, ProviderResult, StateProvider, + StateProviderFactory, +}; +use reth_rpc_api::eth::helpers::FullEthApi; +use reth_rpc_eth_types::EthApiError; +use thiserror::Error; + +/// A request cannot be served by the current durable proof window. +#[derive(Debug, Error, PartialEq, Eq)] +enum ProofWindowError { + #[error("historical proof database is not initialized")] + Uninitialized, + #[error("block {requested} is outside the historical proof window [{earliest}, {latest}]")] + Outside { + requested: u64, + earliest: u64, + latest: u64, + }, + #[error( + "historical proof database is not canonical at block {block}: stored {stored}, canonical {canonical:?}" + )] + CanonicalMismatch { + block: u64, + stored: B256, + canonical: Option, + }, +} + +fn validate_canonical_anchor( + block: u64, + stored: B256, + canonical: Option, +) -> Result<(), ProofWindowError> { + if canonical != Some(stored) { + return Err(ProofWindowError::CanonicalMismatch { + block, + stored, + canonical, + }); + } + Ok(()) +} + +fn validate_window( + requested: u64, + earliest: Option, + latest: Option, +) -> Result<(), ProofWindowError> { + let (Some(earliest), Some(latest)) = (earliest, latest) else { + return Err(ProofWindowError::Uninitialized); + }; + if requested < earliest || requested > latest { + return Err(ProofWindowError::Outside { + requested, + earliest, + latest, + }); + } + Ok(()) +} + +/// Builds providers that source account/storage state only from Morph proof history. +#[derive(Debug)] +pub struct MorphProofStateProviderFactory { + eth_api: Eth, + storage: MorphProofsStorage

, +} + +impl MorphProofStateProviderFactory { + /// Creates a proof state-provider factory. + pub const fn new(eth_api: Eth, storage: MorphProofsStorage

) -> Self { + Self { eth_api, storage } + } +} + +impl<'a, Eth, P> MorphProofStateProviderFactory +where + Eth: FullEthApi + Send + Sync + 'static, + P: MorphProofsStore + Clone + 'a, +{ + /// Returns a provider only when `block_id` is inside the durable proof window. + pub fn state_provider( + &'a self, + block_id: Option, + ) -> ProviderResult> { + let block_id = block_id.unwrap_or_default(); + let block_number = self + .eth_api + .provider() + .block_number_for_id(block_id)? + .ok_or(EthApiError::HeaderNotFound(block_id)) + .map_err(ProviderError::other)?; + if let BlockId::Hash(requested) = block_id { + let canonical_hash = self.eth_api.provider().block_hash(block_number)?; + if canonical_hash != Some(requested.block_hash) { + return Err(ProviderError::other(EthApiError::HeaderNotFound(block_id))); + } + } + + // Pin window validation and all proof cursors to one MDBX snapshot. Otherwise a prune + // committed between the bounds check and proof generation could remap version zero to a + // newer baseline and produce a proof for the wrong state root. + let proof_tx = self + .storage + .ro_tx() + .map_err(|error| ProviderError::Database(error.into()))?; + let earliest = self + .storage + .get_earliest_block_number_with_tx(&proof_tx) + .map_err(|error| ProviderError::Database(error.into()))? + .map(|(number, _)| number); + let latest = self + .storage + .get_latest_block_number_with_tx(&proof_tx) + .map_err(|error| ProviderError::Database(error.into()))?; + validate_window(block_number, earliest, latest.map(|(number, _)| number)) + .map_err(ProviderError::other)?; + + let (latest_number, latest_hash) = + latest.ok_or_else(|| ProviderError::other(ProofWindowError::Uninitialized))?; + let canonical_latest_hash = self.eth_api.provider().block_hash(latest_number)?; + validate_canonical_anchor(latest_number, latest_hash, canonical_latest_hash) + .map_err(ProviderError::other)?; + + // Bytecode is content-addressed and block hashes are canonical-chain data, so a latest + // provider is sufficient for those auxiliary reads. Account/storage trie reads below are + // always routed to proof history; no Reth historical overlay is constructed. + let auxiliary = self.eth_api.provider().latest()?; + Ok(Box::new(MorphProofsStateProviderRef::new_with_tx( + auxiliary, + &self.storage, + block_number, + proof_tx, + ))) + } +} + +#[cfg(test)] +mod tests { + use alloy_primitives::B256; + + use super::{ProofWindowError, validate_canonical_anchor, validate_window}; + + #[test] + fn rejects_an_empty_window() { + assert_eq!( + validate_window(10, None, None), + Err(ProofWindowError::Uninitialized) + ); + } + + #[test] + fn accepts_only_inclusive_window_bounds() { + assert!(validate_window(10, Some(10), Some(20)).is_ok()); + assert!(validate_window(20, Some(10), Some(20)).is_ok()); + assert!(matches!( + validate_window(9, Some(10), Some(20)), + Err(ProofWindowError::Outside { .. }) + )); + assert!(matches!( + validate_window(21, Some(10), Some(20)), + Err(ProofWindowError::Outside { .. }) + )); + } + + #[test] + fn fails_closed_while_proof_history_is_on_a_stale_fork() { + let stored = B256::repeat_byte(0x11); + assert!(validate_canonical_anchor(20, stored, Some(stored)).is_ok()); + assert!(matches!( + validate_canonical_anchor(20, stored, Some(B256::repeat_byte(0x22))), + Err(ProofWindowError::CanonicalMismatch { block: 20, .. }) + )); + assert!(matches!( + validate_canonical_anchor(20, stored, None), + Err(ProofWindowError::CanonicalMismatch { block: 20, .. }) + )); + } +} diff --git a/local-test/README.md b/local-test/README.md index e07d35a..bc48e40 100644 --- a/local-test/README.md +++ b/local-test/README.md @@ -98,7 +98,7 @@ To wipe chain data and start syncing from scratch: ./local-test/reset.sh hoodi --yes # Reset hoodi (no confirmation) ``` -This removes reth's canonical DB, RocksDB history indices, static files, ExEx state, Morph reference index state, and `node-data/data/` for the specified network. Config files (genesis, keys), `reth.toml`, discovery secrets, and logs are preserved. +This removes reth's canonical DB, RocksDB history indices, static files, ExEx state, Morph reference index state, historical proof MDBX state, and `node-data/data/` for the specified network. Config files (genesis, keys), `reth.toml`, discovery secrets, and logs are preserved. ## Storage & engine tuning diff --git a/local-test/reset.sh b/local-test/reset.sh index babc3e9..68298e4 100755 --- a/local-test/reset.sh +++ b/local-test/reset.sh @@ -21,6 +21,7 @@ echo " - ${RETH_DATA_DIR}/rocksdb" echo " - ${RETH_DATA_DIR}/static_files" echo " - ${RETH_DATA_DIR}/exex" echo " - ${RETH_DATA_DIR}/morph" +echo " - ${RETH_DATA_DIR}/historical-proofs" echo " - ${NODE_HOME}/data" echo echo "This keeps:" @@ -45,6 +46,7 @@ rm -rf \ "${RETH_DATA_DIR}/static_files" \ "${RETH_DATA_DIR}/exex" \ "${RETH_DATA_DIR}/morph" \ + "${RETH_DATA_DIR}/historical-proofs" \ "${NODE_HOME}/data" mkdir -p "${RETH_DATA_DIR}" "${NODE_HOME}/data" diff --git a/local-test/reth-start.sh b/local-test/reth-start.sh index 3634f57..2cbc3c0 100755 --- a/local-test/reth-start.sh +++ b/local-test/reth-start.sh @@ -37,7 +37,6 @@ args=( --authrpc.jwtsecret "${JWT_SECRET}" --log.file.directory "$(dirname "${RETH_LOG_FILE}")" --log.file.filter info - --rpc.eth-proof-window 1209600 # Local testing: explicitly avoid external IP probes such as icanhazip.com. --nat none # Batch MDBX writes so they don't compete with Tendermint's LevelDB fsyncs