Upgrade reth 1.9.3 → 1.11.3

Cargo.toml

@@ -1,7 +1,7 @@
 [workspace.package]
 version = "0.1.0"
 edition = "2021"
-rust-version = "1.85"
+rust-version = "1.88"
 license = "MIT OR Apache-2.0"
 homepage = "https://github.com/flashbots/rbuilder"
 repository = "https://github.com/flashbots/rbuilder"
@@ -70,62 +70,62 @@ codegen-units = 1
 incremental = false
 
 [workspace.dependencies]
-reth = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-db = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-db-common = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-node-api = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-node-core = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-ethereum-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-provider = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646", features = [
+reth = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-db = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-db-common = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-node-api = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-node-core = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-ethereum-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-provider = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b", features = [
     "test-utils",
 ] }
-reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-evm = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-revm = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-execution-errors = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-trie-db = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
-
-# compatible with reth 27a8c0f5a6dfb27dea84c5751776ecabdd069646 dependencies
-revm = { version = "31.0.2", features = [
+reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-evm = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-revm = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-execution-errors = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-trie-db = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
+
+# compatible with reth d6324d63e27ef6b7c49cdc9b1977c1b808234c7b dependencies
+revm = { version = "34.0.0", features = [
     "std",
     "secp256k1",
     "optional_balance_check",
 ], default-features = false }
-revm-inspectors = { version = "0.32.0", default-features = false }
+revm-inspectors = { version = "0.34.2", default-features = false }
 
 ethereum_ssz_derive = "0.9.0"
 ethereum_ssz = "0.9.0"
 
-alloy-primitives = { version = "1.4.1", default-features = false, features = [
+alloy-primitives = { version = "1.5.6", default-features = false, features = [
     "getrandom",
 ] }
-alloy-rlp = "0.3.10"
+alloy-rlp = "0.3.13"
 alloy-chains = "0.2.5"
-alloy-trie = { version = "0.8.1", default-features = false }
-alloy-evm = { version = "0.23.3", default-features = false }
-alloy-provider = { version = "1.0.41", features = ["ipc", "pubsub", "ws"] }
-alloy-eips = { version = "1.0.41" }
-alloy-rpc-types = { version = "1.0.41" }
-alloy-json-rpc = { version = "1.0.41" }
-alloy-network = { version = "1.0.41" }
-alloy-network-primitives = { version = "1.0.41" }
-alloy-node-bindings = { version = "1.0.41" }
-alloy-consensus = { version = "1.0.41", features = ["kzg"] }
-alloy-rpc-types-beacon = { version = "1.0.41", features = ["ssz"] }
-alloy-rpc-types-engine = { version = "1.0.41", features = ["ssz"] }
-alloy-rpc-types-eth = { version = "1.0.41" }
-alloy-signer = { version = "1.0.41" }
-alloy-signer-local = { version = "1.0.41" }
+alloy-trie = { version = "0.9.4", default-features = false }
+alloy-evm = { version = "0.27.2", default-features = false }
+alloy-provider = { version = "1.6.3", features = ["ipc", "pubsub", "ws"] }
+alloy-eips = { version = "1.6.3" }
+alloy-rpc-types = { version = "1.6.3" }
+alloy-json-rpc = { version = "1.6.3" }
+alloy-network = { version = "1.6.3" }
+alloy-network-primitives = { version = "1.6.3" }
+alloy-node-bindings = { version = "1.6.3" }
+alloy-consensus = { version = "1.6.3", features = ["kzg"] }
+alloy-rpc-types-beacon = { version = "1.6.3", features = ["ssz"] }
+alloy-rpc-types-engine = { version = "1.6.3", features = ["ssz"] }
+alloy-rpc-types-eth = { version = "1.6.3" }
+alloy-signer = { version = "1.6.3" }
+alloy-signer-local = { version = "1.6.3" }
 
 # Version required by ethereum-consensus beacon-api-client
 mev-share-sse = { git = "https://github.com/paradigmxyz/mev-share-rs", rev = "9eb2b0138ab3202b9eb3af4b19c7b3bf40b0faa8", default-features = false }

crates/eth-sparse-mpt/Cargo.toml

@@ -17,7 +17,7 @@ rustc-hash = "2.0.0"
 rayon = "1.10.0"
 smallvec = "1.13.2"
 alloy-trie.workspace = true
-nybbles = { version = "0.3.3", features = ["serde"] }
+nybbles = { version = "0.4", features = ["serde"] }
 
 tracing.workspace = true
 
@@ -36,6 +36,9 @@ reth-trie-db.workspace = true
 reth-provider.workspace = true
 reth-revm = { workspace = true, optional = true }
 
+# Only used by the dev-tools harness binaries to open reth's RocksDB read-only.
+rbuilder-utils = { workspace = true, optional = true }
+
 # revm
 revm.workspace = true
 
@@ -65,6 +68,7 @@ dev-tools = [
     "dep:reth-node-ethereum",
     "dep:reth-trie-parallel",
     "dep:reth-revm",
+    "dep:rbuilder-utils",
 ]
 
 [dev-dependencies]

crates/eth-sparse-mpt/benches/trie_do_bench.rs

@@ -92,14 +92,14 @@ fn insert_proofs(c: &mut Criterion) {
     let mut proofs: HashMap<Nibbles, Vec<(Nibbles, Bytes)>> = Default::default();
     for key in &byte_keys {
         let key = Nibbles::unpack(key);
-        let current_key_proofs = proofs.entry(key.clone()).or_default();
+        let current_key_proofs = proofs.entry(key).or_default();
         for (path, node) in &nodes {
             if key.starts_with(path) {
-                current_key_proofs.push((path.clone(), node.clone().into()));
+                current_key_proofs.push((*path, node.clone().into()));
             }
         }
-        current_key_proofs.sort_by_key(|(p, _)| p.clone());
-        current_key_proofs.dedup_by_key(|(p, _)| p.clone());
+        current_key_proofs.sort_by_key(|(p, _)| *p);
+        current_key_proofs.dedup_by_key(|(p, _)| *p);
     }
     for (path, proof) in proofs {
         proof_store.add_proof(path, proof).unwrap();

crates/eth-sparse-mpt/benches/trie_insert_bench.rs

@@ -120,8 +120,8 @@ fn gather_nodes(c: &mut Criterion) {
             .into_iter()
             .flat_map(|mp| mp.account_subtree.into_iter().collect::<Vec<_>>())
             .collect();
-        account_proof.sort_by_key(|(p, _)| p.clone());
-        account_proof.dedup_by_key(|(p, _)| p.clone());
+        account_proof.sort_by_key(|(p, _)| *p);
+        account_proof.dedup_by_key(|(p, _)| *p);
         account_proof
     };
 

crates/eth-sparse-mpt/src/bin/cache-warm-compare-v-experimental.rs

@@ -479,7 +479,7 @@ fn calc_stats(data: &[Duration]) -> Result<Stats> {
     ms.sort_by(|a, b| a.total_cmp(b));
 
     let mean_ms = ms.iter().sum::<f64>() / ms.len() as f64;
-    let median_ms = if ms.len() % 2 == 0 {
+    let median_ms = if ms.len().is_multiple_of(2) {
         (ms[ms.len() / 2 - 1] + ms[ms.len() / 2]) / 2.0
     } else {
         ms[ms.len() / 2]

crates/eth-sparse-mpt/src/bin/correctness-harness.rs

@@ -14,12 +14,13 @@ use reth_evm_ethereum::EthEvmConfig;
 use reth_node_api::NodeTypesWithDBAdapter;
 use reth_node_ethereum::EthereumNode;
 use reth_provider::{
-    providers::{ConsistentDbView, OverlayStateProviderFactory, StaticFileProvider},
+    providers::{ConsistentDbView, OverlayStateProviderFactory, StaticFileProviderBuilder},
     BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, HeaderProvider,
     ProviderFactory, TransactionVariant,
 };
 use reth_revm::database::StateProviderDatabase;
 use reth_trie::TrieInput;
+use reth_trie_db::ChangesetCache;
 use reth_trie_parallel::root::ParallelStateRoot;
 use revm::database::BundleState;
 use std::{
@@ -347,17 +348,33 @@ fn calculate_reth_root(
     parent_hash: B256,
     outcome: &BundleState,
 ) -> Result<B256> {
-    let overlay = OverlayStateProviderFactory::new(factory.clone());
+    let overlay = OverlayStateProviderFactory::new(factory.clone(), ChangesetCache::new());
     let hasher = factory
         .history_by_block_hash(parent_hash)
         .with_context(|| format!("failed to open state provider at parent hash {parent_hash:?}"))?;
     let hashed_post_state = hasher.hashed_post_state(outcome);
     let trie_input = TrieInput::from_state(hashed_post_state);
-    ParallelStateRoot::new(overlay, trie_input.prefix_sets.freeze())
+    ParallelStateRoot::new(overlay, trie_input.prefix_sets.freeze(), task_runtime())
         .incremental_root()
         .with_context(|| "parallel state root failed")
 }
 
+/// Process-wide reth task runtime for parallel provider/trie I/O. Built once and cached: the reth
+/// `Runtime` owns rayon thread pools, so it must not be rebuilt per call. `Runtime::default` is
+/// gated behind reth's `test-utils` feature, so build it explicitly here. The harness `main` is
+/// synchronous, so a standalone runtime (no ambient tokio handle) is what we want.
+fn task_runtime() -> rbuilder_utils::tasks::Runtime {
+    use rbuilder_utils::tasks::{Runtime, RuntimeBuilder, RuntimeConfig};
+    static RUNTIME: std::sync::OnceLock<Runtime> = std::sync::OnceLock::new();
+    RUNTIME
+        .get_or_init(|| {
+            RuntimeBuilder::new(RuntimeConfig::default())
+                .build()
+                .expect("failed to build reth task runtime")
+        })
+        .clone()
+}
+
 fn run_unwind_once(cli: &Cli, iteration: usize) -> Result<()> {
     let output = Command::new(&cli.reth_bin)
         .arg("stage")
@@ -390,6 +407,7 @@ fn open_provider_factory(
 ) -> Result<Factory> {
     let db_path = datadir.join("db");
     let static_files_path = datadir.join("static_files");
+    let rocksdb_path = datadir.join("rocksdb");
     let chain_spec = parse_chain_spec(chain)?;
 
     let db = Arc::new(
@@ -403,24 +421,32 @@ fn open_provider_factory(
         .with_context(|| format!("failed to open reth db at {}", db_path.display()))?,
     );
 
-    let static_files =
-        StaticFileProvider::read_only(&static_files_path, false).with_context(|| {
+    // Configuring blocks-per-file now goes through `StaticFileProviderBuilder` (the inherent
+    // `with_custom_blocks_per_file` on `StaticFileProvider` was replaced by the builder's
+    // `with_blocks_per_file`, which sets the value across all segments).
+    let static_files = {
+        let builder = StaticFileProviderBuilder::read_only(&static_files_path);
+        let builder = if let Some(blocks_per_file) = static_file_blocks_per_file {
+            builder.with_blocks_per_file(blocks_per_file)
+        } else {
+            builder
+        };
+        builder.build().with_context(|| {
             format!(
                 "failed to open static files at {}",
                 static_files_path.display()
             )
-        })?;
-    let static_files = if let Some(blocks_per_file) = static_file_blocks_per_file {
-        static_files.with_custom_blocks_per_file(blocks_per_file)
-    } else {
-        static_files
+        })?
     };
 
-    Ok(ProviderFactory::<NodeTypes>::new(
+    ProviderFactory::<NodeTypes>::new(
         db,
         chain_spec,
         static_files,
-    ))
+        rbuilder_utils::reth_db::open_rocksdb_read_only(&rocksdb_path)?,
+        Default::default(),
+    )
+    .with_context(|| "failed to create provider factory")
 }
 
 fn parse_chain_spec(chain: &str) -> Result<Arc<ChainSpec>> {

crates/eth-sparse-mpt/src/utils.rs

@@ -29,25 +29,23 @@ pub fn rlp_pointer(rlp_encode: Bytes) -> Bytes {
 }
 
 pub fn concat_path(p1: &Nibbles, p2: &[u8]) -> Nibbles {
-    let mut result = Nibbles::with_capacity(p1.len() + p2.len());
-    result.extend_from_slice_unchecked(p1);
+    let mut result = *p1;
     result.extend_from_slice_unchecked(p2);
     result
 }
 
 pub fn strip_first_nibble_mut(p: &mut Nibbles) -> u8 {
-    let nibble = p[0];
-    let vec = p.as_mut_vec_unchecked();
-    vec.remove(0);
+    let nibble = p.get_unchecked(0);
+    *p = p.slice_unchecked(1, p.len());
     nibble
 }
 
 #[inline]
 pub fn extract_prefix_and_suffix(p1: &Nibbles, p2: &Nibbles) -> (Nibbles, Nibbles, Nibbles) {
     let prefix_len = p1.common_prefix_length(p2);
-    let prefix = Nibbles::from_nibbles_unchecked(&p1[..prefix_len]);
-    let suffix1 = Nibbles::from_nibbles_unchecked(&p1[prefix_len..]);
-    let suffix2 = Nibbles::from_nibbles_unchecked(&p2[prefix_len..]);
+    let prefix = p1.slice_unchecked(0, prefix_len);
+    let suffix1 = p1.slice_unchecked(prefix_len, p1.len());
+    let suffix2 = p2.slice_unchecked(prefix_len, p2.len());
 
     (prefix, suffix1, suffix2)
 }
@@ -136,16 +134,3 @@ fn mismatch_chunks<const N: usize>(xs: &[u8], ys: &[u8]) -> usize {
         .take_while(|(x, y)| x == y)
         .count()
 }
-
-// rbuilder uses nybbles v3.3.0 and reth_trie uses nybbles v4.3.0. This is a temporary fix to convert between the two.
-// We can remove the below methods once rbuilder has been upgraded to nybbles v4.3.0.
-// nybbles v4.3.0 has a breaking change (byte array with 1 byte per nybble vs U256 packed data + length) in the API which breaks a lot of parts of the eth sparse trie code.
-#[inline]
-pub fn convert_reth_nybbles_to_nibbles(n: reth_trie::Nibbles) -> Nibbles {
-    Nibbles::from_nibbles(n.to_vec())
-}
-
-#[inline]
-pub fn convert_nibbles_to_reth_nybbles(n: Nibbles) -> reth_trie::Nibbles {
-    reth_trie::Nibbles::from_nibbles(n.as_slice())
-}

crates/eth-sparse-mpt/src/v1/reth_sparse_trie/shared_cache.rs

@@ -146,11 +146,11 @@ impl RethSparseTrieShareCacheInternal {
         multiproof: MultiProof,
     ) -> Result<(), AddNodeError> {
         let mut nodes: Vec<_> = multiproof.account_subtree.into_iter().collect();
-        nodes.sort_by_key(|(p, _)| p.clone());
+        nodes.sort_by_key(|(p, _)| *p);
         self.account_trie.add_nodes(&nodes)?;
         for (account, storge_proofs) in multiproof.storages {
             let mut nodes: Vec<_> = storge_proofs.subtree.into_iter().collect();
-            nodes.sort_by_key(|(p, _)| p.clone());
+            nodes.sort_by_key(|(p, _)| *p);
             let account = Bytes::copy_from_slice(account.as_slice());
             let storage_trie = self.storage_tries.entry(account).or_default();
             storage_trie.add_nodes(&nodes)?;