From d924c1c2f43dd29e63cd299ffbae733ff16b686b Mon Sep 17 00:00:00 2001 From: ljedrz Date: Fri, 31 Jul 2026 12:31:12 +0200 Subject: [PATCH 1/2] perf: speed up block tree (de)serialization Signed-off-by: ljedrz --- console/collections/Cargo.toml | 3 + console/collections/benches/merkle_tree.rs | 114 ++++++++++- console/collections/src/merkle_tree/mod.rs | 100 ++++++++- .../collections/src/merkle_tree/tests/mod.rs | 1 + .../src/merkle_tree/tests/state.rs | 189 ++++++++++++++++++ console/network/src/canary_v0.rs | 8 + console/network/src/lib.rs | 8 + console/network/src/mainnet_v0.rs | 8 + console/network/src/testnet_v0.rs | 8 + ledger/store/src/block/mod.rs | 11 +- ledger/store/src/helpers/rocksdb/block.rs | 30 ++- 11 files changed, 464 insertions(+), 16 deletions(-) create mode 100644 console/collections/src/merkle_tree/tests/state.rs diff --git a/console/collections/Cargo.toml b/console/collections/Cargo.toml index c3200d3632..4f9cea87bd 100644 --- a/console/collections/Cargo.toml +++ b/console/collections/Cargo.toml @@ -44,6 +44,9 @@ workspace = true [dev-dependencies.snarkvm-console-network] path = "../network" +[dev-dependencies.bincode] +workspace = true + [dev-dependencies.criterion] workspace = true diff --git a/console/collections/benches/merkle_tree.rs b/console/collections/benches/merkle_tree.rs index 7c97a0af41..c210419593 100644 --- a/console/collections/benches/merkle_tree.rs +++ b/console/collections/benches/merkle_tree.rs @@ -16,7 +16,12 @@ #[macro_use] extern crate criterion; +use snarkvm_console_algorithms::{BHP512, BHP1024}; +use snarkvm_console_collections::merkle_tree::MerkleTreeState; use snarkvm_console_network::{ + BHP_512, + BHP_1024, + BHPMerkleTree, MainnetV0, Network, prelude::{Rng, TestRng, ToBits, Uniform}, @@ -24,7 +29,7 @@ use snarkvm_console_network::{ use snarkvm_console_types::Field; use criterion::{BatchSize, BenchmarkId, Criterion}; -use std::collections::BTreeMap; +use std::{collections::BTreeMap, time::Duration}; const DEPTH: u8 = 32; const MAX_INSTANTIATED_DEPTH: u8 = 8; @@ -33,6 +38,10 @@ const NUM_LEAVES: &[usize] = &[1, 100, 10_000]; const APPEND_SIZES: &[usize] = &[1, 100, 10_000]; const UPDATE_SIZES: &[usize] = &[1, 100, 1_000]; +/// The tree sizes used by the `MerkleTreeState` benchmarks; these reach further +/// than `NUM_LEAVES`, as caching a tree is most interesting for large trees. +const STATE_NUM_LEAVES: &[usize] = &[1, 100, 10_000, 1_000_000]; + /// Generates the specified number of random Merkle tree leaves. macro_rules! generate_leaves { ($num_leaves:expr, $rng:expr) => {{ (0..$num_leaves).map(|_| Field::::rand($rng).to_bits_le()).collect::>() }}; @@ -175,9 +184,110 @@ fn update_vs_update_many(c: &mut Criterion) { } } +/// A Merkle tree as it was cached before [`MerkleTreeState`] was introduced, i.e. the +/// whole tree, hashers included. +/// +/// Note: `bincode` encodes a tuple and a struct identically - as the concatenation of +/// their fields - so this decodes the legacy payload byte for byte. +type LegacyCachedTree<'a> = (BHP1024, BHP512, MerkleTreeState<'a, MainnetV0>); + +/// Produces the payload that caching the given tree used to write, for comparison. +fn legacy_payload(merkle_tree: &BHPMerkleTree) -> Vec { + bincode::serialize(&(&*BHP_1024, &*BHP_512, merkle_tree.to_state())).unwrap() +} + +/// Compares recreating a Merkle tree from a cached [`MerkleTreeState`] against +/// constructing the same tree from scratch, across a range of tree sizes. +/// +/// Run with `cargo bench --bench merkle_tree -- MerkleTreeState`. +fn state(c: &mut Criterion) { + let mut group = c.benchmark_group("MerkleTreeState"); + let mut rng = TestRng::default(); + // Accumulate leaves in a vector to avoid recomputing across iterations. + let leaves = generate_leaves!(*STATE_NUM_LEAVES.last().unwrap(), &mut rng); + + for num_leaves in STATE_NUM_LEAVES { + // Construct a Merkle tree with the specified number of leaves, and cache its state. + let merkle_tree = MainnetV0::merkle_tree_bhp::(&leaves[..*num_leaves]).unwrap(); + let cached = bincode::serialize(&merkle_tree.to_state()).unwrap(); + // Report the size of the cached payload against that of the legacy one; the former + // should be proportional to the tree, i.e. it must not contain the hashers. + println!( + "MerkleTreeState/{num_leaves} leaves: {} bytes cached ({} bytes in the legacy format)", + cached.len(), + legacy_payload(&merkle_tree).len() + ); + + // For reference: constructing the tree from scratch, i.e. not using a cache at all. + group.bench_with_input(BenchmarkId::new("from_scratch", num_leaves), num_leaves, |b, num_leaves| { + b.iter(|| MainnetV0::merkle_tree_bhp::(&leaves[..*num_leaves]).unwrap()) + }); + // Serializing the state of the tree, i.e. the cost of writing the cache. + group.bench_with_input(BenchmarkId::new("serialize", num_leaves), &merkle_tree, |b, merkle_tree| { + b.iter(|| bincode::serialize(&merkle_tree.to_state()).unwrap()) + }); + // Deserializing the state, i.e. the decoding half of the cost of reading the cache. + group.bench_with_input(BenchmarkId::new("deserialize", num_leaves), &cached, |b, cached| { + b.iter(|| bincode::deserialize::>(cached).unwrap()) + }); + // Recreating the tree from a deserialized state, i.e. the validating half. + group.bench_with_input(BenchmarkId::new("from_state", num_leaves), &cached, |b, cached| { + b.iter_batched( + || bincode::deserialize::>(cached).unwrap(), + |state| MainnetV0::merkle_tree_bhp_from_state::(state).unwrap(), + BatchSize::LargeInput, + ) + }); + // Both halves together, i.e. what a node pays to load a cached tree. + group.bench_with_input(BenchmarkId::new("deserialize_and_recreate", num_leaves), &cached, |b, cached| { + b.iter(|| { + let state = bincode::deserialize::>(cached).unwrap(); + MainnetV0::merkle_tree_bhp_from_state::(state).unwrap() + }) + }); + } + + group.finish(); +} + +/// Measures loading a tree from a legacy cache payload, i.e. one that carries the hashers. +/// +/// This is kept apart from [`state`] because a single iteration takes tens of seconds: the +/// hashers hold hundreds of thousands of group elements, and deserializing each one costs a +/// subgroup check, i.e. a full scalar multiplication. That cost is also invariant in the +/// size of the tree, which is why this is measured for a single tree size only. +/// +/// Run with `cargo bench --bench merkle_tree -- LegacyMerkleTreeCache` (slow). +fn legacy_state(c: &mut Criterion) { + let mut group = c.benchmark_group("LegacyMerkleTreeCache"); + let mut rng = TestRng::default(); + + let num_leaves = STATE_NUM_LEAVES[0]; + let leaves = generate_leaves!(num_leaves, &mut rng); + let merkle_tree = MainnetV0::merkle_tree_bhp::(&leaves).unwrap(); + let legacy = legacy_payload(&merkle_tree); + println!("LegacyMerkleTreeCache/{num_leaves} leaves: {} bytes cached", legacy.len()); + + group.bench_with_input(BenchmarkId::new("deserialize", num_leaves), &legacy, |b, legacy| { + b.iter(|| bincode::deserialize::>(legacy).unwrap()) + }); + + group.finish(); +} + criterion_group! { name = merkle_tree; config = Criterion::default().sample_size(10); targets = new, append, update, update_many, update_vs_update_many } -criterion_main!(merkle_tree); +criterion_group! { + name = merkle_tree_state; + config = Criterion::default().sample_size(10).warm_up_time(Duration::from_secs(1)); + targets = state +} +criterion_group! { + name = legacy_merkle_tree_cache; + config = Criterion::default().sample_size(10).warm_up_time(Duration::from_secs(1)); + targets = legacy_state +} +criterion_main!(merkle_tree, merkle_tree_state, legacy_merkle_tree_cache); diff --git a/console/collections/src/merkle_tree/mod.rs b/console/collections/src/merkle_tree/mod.rs index 3ac69fa1b3..8bf77f2b74 100644 --- a/console/collections/src/merkle_tree/mod.rs +++ b/console/collections/src/merkle_tree/mod.rs @@ -31,7 +31,7 @@ use locktick::parking_lot::Mutex; #[cfg(not(feature = "locktick"))] use parking_lot::Mutex; use serde::{Deserialize, Serialize}; -use std::{collections::BTreeMap, mem}; +use std::{borrow::Cow, collections::BTreeMap, mem}; #[cfg(not(feature = "serial"))] use rayon::prelude::*; @@ -50,8 +50,6 @@ use rayon::prelude::*; /// Padding levels are then added as needed to reach the full `DEPTH`, each of /// which is constructed by hashing the root of the previous level together with /// `e`. -#[derive(Deserialize, Serialize)] -#[serde(bound = "E: Serialize + DeserializeOwned, LH: Serialize + DeserializeOwned, PH: Serialize + DeserializeOwned")] pub struct MerkleTree, PH: PathHash>, const DEPTH: u8> { /// The leaf hasher for the Merkle tree. leaf_hasher: LH, @@ -66,10 +64,30 @@ pub struct MerkleTree, PH: PathHas /// The number of hashed leaves in the tree. number_of_leaves: usize, /// An optimization: the previous tree allocation reused in prepare_append. - #[serde(skip)] preserved_tree_allocation: Mutex>>, } +/// The contents of a [`MerkleTree`], sans its hashers. +/// +/// This is the serializable form of a Merkle tree, intended for caching one on +/// disk. Note that a [`MerkleTree`] itself is deliberately **not** serializable: +/// its hashers hold precomputed bases (tens of MiBs of group elements for the +/// BHP hashers), and deserializing those is orders of magnitude more expensive +/// than setting them up from scratch - each group element costs a subgroup check, +/// i.e. a full scalar multiplication. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(bound = "E: Serialize + DeserializeOwned")] +pub struct MerkleTreeState<'a, E: Environment> { + /// The computed root of the full Merkle tree. + root: Field, + /// The internal hashes, from root to hashed leaves, of the full Merkle tree. + tree: Cow<'a, [Field]>, + /// The canonical empty hash. + empty_hash: Field, + /// The number of hashed leaves in the tree. + number_of_leaves: usize, +} + impl, PH: PathHash>, const DEPTH: u8> Clone for MerkleTree { @@ -178,6 +196,80 @@ impl, PH: PathHash }) } + /// Returns the contents of the Merkle tree, sans its hashers. + /// + /// This borrows from the tree, so it is cheap even for very large trees; use + /// [`Self::from_state`] to recreate the tree from the returned state. + pub fn to_state(&self) -> MerkleTreeState<'_, E> { + MerkleTreeState { + root: self.root, + tree: Cow::Borrowed(&self.tree), + empty_hash: self.empty_hash, + number_of_leaves: self.number_of_leaves, + } + } + + /// Recreates a Merkle tree from the given state, using the given hashers. + /// + /// The state is checked for internal consistency, which includes recomputing + /// the root from the topmost node; since only the padding levels are hashed, + /// this is cheap. Note that this cannot attest that the tree corresponds to + /// any particular set of leaves, so the caller is still expected to check the + /// resulting root against a trusted value. + pub fn from_state(leaf_hasher: &LH, path_hasher: &PH, state: MerkleTreeState<'_, E>) -> Result { + // Ensure the Merkle tree depth is greater than 0. + ensure!(DEPTH > 0, "Merkle tree depth must be greater than 0"); + // Ensure the Merkle tree depth is less than or equal to 64. + ensure!(DEPTH <= 64u8, "Merkle tree depth must be less than or equal to 64"); + + let MerkleTreeState { root, tree, empty_hash, number_of_leaves } = state; + // Note: this is a no-op if the state was deserialized, as opposed to borrowed. + let tree = tree.into_owned(); + + // Ensure the empty hash matches the one produced by the given path hasher; a + // mismatch means that the state was produced for a different network or hasher. + ensure!(empty_hash == path_hasher.hash_empty()?, "The Merkle tree state has an invalid empty hash"); + + // Compute the maximum number of leaves. + let max_leaves = match number_of_leaves.checked_next_power_of_two() { + Some(num_leaves) => num_leaves, + None => bail!("Integer overflow when computing the maximum number of leaves in the Merkle tree"), + }; + // Compute the number of nodes. + let num_nodes = max_leaves - 1; + // Compute the number of padded levels. + let padding_depth = DEPTH - tree_depth::(max_leaves + num_nodes)?; + + // Ensure the tree contains exactly as many nodes as its number of leaves implies. + let minimum_tree_size = std::cmp::max( + 1, + num_nodes + number_of_leaves + if number_of_leaves > 1 { number_of_leaves % 2 } else { 0 }, + ); + ensure!( + tree.len() == minimum_tree_size, + "The Merkle tree state contains {} nodes, expected {minimum_tree_size} for {number_of_leaves} leaves", + tree.len() + ); + + // Recompute the root hash, by iterating from the root level up to `DEPTH`. + let mut root_hash = tree[0]; + for _ in 0..padding_depth { + // Update the root hash, by hashing the current root hash with the empty hash. + root_hash = path_hasher.hash_children(&root_hash, &empty_hash)?; + } + ensure!(root_hash == root, "The Merkle tree state has an invalid root"); + + Ok(Self { + leaf_hasher: leaf_hasher.clone(), + path_hasher: path_hasher.clone(), + root, + tree, + empty_hash, + number_of_leaves, + preserved_tree_allocation: Default::default(), + }) + } + #[inline] /// Returns a new Merkle tree with the given new leaves appended to it. pub fn prepare_append(&self, new_leaves: &[LH::Leaf]) -> Result { diff --git a/console/collections/src/merkle_tree/tests/mod.rs b/console/collections/src/merkle_tree/tests/mod.rs index 563945f103..b81d6490f8 100644 --- a/console/collections/src/merkle_tree/tests/mod.rs +++ b/console/collections/src/merkle_tree/tests/mod.rs @@ -17,6 +17,7 @@ use super::*; mod append; mod remove; +mod state; mod test_print; mod update; mod update_many; diff --git a/console/collections/src/merkle_tree/tests/state.rs b/console/collections/src/merkle_tree/tests/state.rs new file mode 100644 index 0000000000..4bb960999d --- /dev/null +++ b/console/collections/src/merkle_tree/tests/state.rs @@ -0,0 +1,189 @@ +// Copyright (c) 2019-2026 Provable Inc. +// This file is part of the snarkVM library. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use snarkvm_console_algorithms::{BHP512, BHP1024, Poseidon}; +use snarkvm_console_types::prelude::Console; + +type CurrentEnvironment = Console; + +const DEPTH: u8 = 8; + +/// Runs the following test: +/// 1. Construct a Merkle tree for the given leaves. +/// 2. Round-trip its state through bincode, and recreate the tree from it. +/// 3. Check that the recreated tree is equivalent to the original one. +fn check_state_round_trip< + E: Environment, + LH: LeafHash, + PH: PathHash>, + const DEPTH: u8, +>( + leaf_hasher: &LH, + path_hasher: &PH, + leaves: &[LH::Leaf], +) -> Result<()> { + let merkle_tree = MerkleTree::::new(leaf_hasher, path_hasher, leaves)?; + + // Round-trip the state of the tree. + let serialized = bincode::serialize(&merkle_tree.to_state())?; + let state: MerkleTreeState = bincode::deserialize(&serialized)?; + let recreated = MerkleTree::::from_state(leaf_hasher, path_hasher, state)?; + + // Ensure the recreated tree matches the original one. + assert_eq!(merkle_tree.root(), recreated.root()); + assert_eq!(merkle_tree.tree(), recreated.tree()); + assert_eq!(merkle_tree.number_of_leaves, recreated.number_of_leaves); + assert_eq!(merkle_tree.empty_hash, recreated.empty_hash); + + // Ensure the recreated tree is usable, i.e. its hashers are functional. + for (leaf_index, leaf) in leaves.iter().enumerate() { + let proof = recreated.prove(leaf_index, leaf)?; + assert!(proof.verify(leaf_hasher, path_hasher, recreated.root(), leaf)); + } + + Ok(()) +} + +#[test] +fn test_state_round_trip_bhp() -> Result<()> { + let mut rng = TestRng::default(); + + let leaf_hasher = BHP1024::::setup("MerkleTreeTest0")?; + let path_hasher = BHP512::::setup("MerkleTreeTest1")?; + + // Check a range of tree sizes, including the empty tree and both parities. + for num_leaves in [0, 1, 2, 3, 4, 7, 8, 100] { + let leaves: Vec> = + (0..num_leaves).map(|_| Field::::rand(&mut rng).to_bits_le()).collect(); + check_state_round_trip::(&leaf_hasher, &path_hasher, &leaves)?; + } + + Ok(()) +} + +#[test] +fn test_state_round_trip_poseidon() -> Result<()> { + let mut rng = TestRng::default(); + + let leaf_hasher = Poseidon::::setup("MerkleTreeTest0")?; + let path_hasher = Poseidon::::setup("MerkleTreeTest1")?; + + for num_leaves in [0, 1, 2, 3, 4, 7, 8, 100] { + let leaves: Vec>> = + (0..num_leaves).map(|_| vec![Field::::rand(&mut rng)]).collect(); + check_state_round_trip::(&leaf_hasher, &path_hasher, &leaves)?; + } + + Ok(()) +} + +#[test] +fn test_state_round_trip_after_append() -> Result<()> { + let mut rng = TestRng::default(); + + let leaf_hasher = BHP1024::::setup("MerkleTreeTest0")?; + let path_hasher = BHP512::::setup("MerkleTreeTest1")?; + + // Grow the tree one leaf at a time, round-tripping the state at every size, as + // `append` maintains the tree size invariant that `from_state` checks. + let mut merkle_tree = MerkleTree::::new(&leaf_hasher, &path_hasher, &[])?; + for _ in 0..20 { + let leaf = Field::::rand(&mut rng).to_bits_le(); + merkle_tree.append(&[leaf])?; + + let serialized = bincode::serialize(&merkle_tree.to_state())?; + let state: MerkleTreeState = bincode::deserialize(&serialized)?; + let recreated = MerkleTree::::from_state(&leaf_hasher, &path_hasher, state)?; + + assert_eq!(merkle_tree.root(), recreated.root()); + assert_eq!(merkle_tree.tree(), recreated.tree()); + } + + Ok(()) +} + +#[test] +fn test_state_does_not_contain_the_hashers() -> Result<()> { + let mut rng = TestRng::default(); + + let leaf_hasher = BHP1024::::setup("MerkleTreeTest0")?; + let path_hasher = BHP512::::setup("MerkleTreeTest1")?; + + // The BHP hashers hold tens of MiBs of precomputed bases, and deserializing a single + // group element costs a subgroup check; including them in the payload used to make + // loading a cached tree take minutes, regardless of how small the tree was. + let leaf_hasher_size = bincode::serialize(&leaf_hasher)?.len(); + let path_hasher_size = bincode::serialize(&path_hasher)?.len(); + + let leaves: Vec> = (0..10).map(|_| Field::::rand(&mut rng).to_bits_le()).collect(); + let merkle_tree = MerkleTree::::new(&leaf_hasher, &path_hasher, &leaves)?; + + // Ensure the payload is a small multiple of the tree's own contents, which also rules + // out either hasher having snuck into it. + let size = bincode::serialize(&merkle_tree.to_state())?.len(); + let tree_size = merkle_tree.tree().len() * Field::::size_in_bytes(); + assert!(size < 2 * tree_size, "the cached state ({size} B) is disproportionate to the tree ({tree_size} B)"); + assert!(size < leaf_hasher_size.min(path_hasher_size)); + + Ok(()) +} + +#[test] +fn test_state_rejects_a_mismatched_path_hasher() -> Result<()> { + let mut rng = TestRng::default(); + + let leaf_hasher = BHP1024::::setup("MerkleTreeTest0")?; + let path_hasher = BHP512::::setup("MerkleTreeTest1")?; + // A path hasher set up with a different domain produces a different empty hash. + let other_path_hasher = BHP512::::setup("MerkleTreeTest2")?; + + let leaves: Vec> = (0..10).map(|_| Field::::rand(&mut rng).to_bits_le()).collect(); + let merkle_tree = MerkleTree::::new(&leaf_hasher, &path_hasher, &leaves)?; + + assert!( + MerkleTree::::from_state( + &leaf_hasher, + &other_path_hasher, + merkle_tree.to_state() + ) + .is_err() + ); + + Ok(()) +} + +#[test] +fn test_state_rejects_a_corrupted_tree() -> Result<()> { + let mut rng = TestRng::default(); + + let leaf_hasher = BHP1024::::setup("MerkleTreeTest0")?; + let path_hasher = BHP512::::setup("MerkleTreeTest1")?; + + let leaves: Vec> = (0..10).map(|_| Field::::rand(&mut rng).to_bits_le()).collect(); + let merkle_tree = MerkleTree::::new(&leaf_hasher, &path_hasher, &leaves)?; + + // A tampered topmost node no longer hashes up to the cached root. + let mut state = merkle_tree.to_state(); + state.tree.to_mut()[0] = Field::::rand(&mut rng); + assert!(MerkleTree::::from_state(&leaf_hasher, &path_hasher, state).is_err()); + + // A truncated tree no longer matches the expected size for its number of leaves. + let mut state = merkle_tree.to_state(); + state.tree.to_mut().pop(); + assert!(MerkleTree::::from_state(&leaf_hasher, &path_hasher, state).is_err()); + + Ok(()) +} diff --git a/console/network/src/canary_v0.rs b/console/network/src/canary_v0.rs index c6dfd55a65..450b7bd7a9 100644 --- a/console/network/src/canary_v0.rs +++ b/console/network/src/canary_v0.rs @@ -651,6 +651,14 @@ impl Network for CanaryV0 { MerkleTree::new(&*CANARY_BHP_1024, &*CANARY_BHP_512, leaves) } + /// Recreates a Merkle tree with a BHP leaf hasher of 1024-bits and a BHP path hasher of + /// 512-bits from the given state, e.g. one that was previously cached on disk. + fn merkle_tree_bhp_from_state( + state: MerkleTreeState<'_, Self>, + ) -> Result> { + MerkleTree::from_state(&*CANARY_BHP_1024, &*CANARY_BHP_512, state) + } + /// Returns a Merkle tree with a Poseidon leaf hasher with input rate of 4 and a Poseidon path hasher with input rate of 2. fn merkle_tree_psd(leaves: &[Vec>]) -> Result> { MerkleTree::new(&*CANARY_POSEIDON_4, &*CANARY_POSEIDON_2, leaves) diff --git a/console/network/src/lib.rs b/console/network/src/lib.rs index f3ed7c3190..7231aab3e6 100644 --- a/console/network/src/lib.rs +++ b/console/network/src/lib.rs @@ -74,6 +74,8 @@ use snarkvm_curves::PairingEngine; use indexmap::IndexMap; use std::sync::{Arc, OnceLock}; +pub use snarkvm_console_collections::merkle_tree::MerkleTreeState; + /// A helper type for the BHP Merkle tree. pub type BHPMerkleTree = MerkleTree, BHP512, DEPTH>; /// A helper type for the Poseidon Merkle tree. @@ -587,6 +589,12 @@ pub trait Network: /// Returns a Merkle tree with a BHP leaf hasher of 1024-bits and a BHP path hasher of 512-bits. fn merkle_tree_bhp(leaves: &[Vec]) -> Result>; + /// Recreates a Merkle tree with a BHP leaf hasher of 1024-bits and a BHP path hasher of + /// 512-bits from the given state, e.g. one that was previously cached on disk. + fn merkle_tree_bhp_from_state( + state: MerkleTreeState<'_, Self>, + ) -> Result>; + /// Returns a Merkle tree with a Poseidon leaf hasher with input rate of 4 and a Poseidon path hasher with input rate of 2. fn merkle_tree_psd(leaves: &[Vec>]) -> Result>; diff --git a/console/network/src/mainnet_v0.rs b/console/network/src/mainnet_v0.rs index a2b3ded30b..6c9aba306e 100644 --- a/console/network/src/mainnet_v0.rs +++ b/console/network/src/mainnet_v0.rs @@ -670,6 +670,14 @@ impl Network for MainnetV0 { MerkleTree::new(&*BHP_1024, &*BHP_512, leaves) } + /// Recreates a Merkle tree with a BHP leaf hasher of 1024-bits and a BHP path hasher of + /// 512-bits from the given state, e.g. one that was previously cached on disk. + fn merkle_tree_bhp_from_state( + state: MerkleTreeState<'_, Self>, + ) -> Result> { + MerkleTree::from_state(&*BHP_1024, &*BHP_512, state) + } + /// Returns a Merkle tree with a Poseidon leaf hasher with input rate of 4 and a Poseidon path hasher with input rate of 2. fn merkle_tree_psd(leaves: &[Vec>]) -> Result> { MerkleTree::new(&*POSEIDON_4, &*POSEIDON_2, leaves) diff --git a/console/network/src/testnet_v0.rs b/console/network/src/testnet_v0.rs index d45bb38459..9bcdf28f9e 100644 --- a/console/network/src/testnet_v0.rs +++ b/console/network/src/testnet_v0.rs @@ -653,6 +653,14 @@ impl Network for TestnetV0 { MerkleTree::new(&*TESTNET_BHP_1024, &*TESTNET_BHP_512, leaves) } + /// Recreates a Merkle tree with a BHP leaf hasher of 1024-bits and a BHP path hasher of + /// 512-bits from the given state, e.g. one that was previously cached on disk. + fn merkle_tree_bhp_from_state( + state: MerkleTreeState<'_, Self>, + ) -> Result> { + MerkleTree::from_state(&*TESTNET_BHP_1024, &*TESTNET_BHP_512, state) + } + /// Returns a Merkle tree with a Poseidon leaf hasher with input rate of 4 and a Poseidon path hasher with input rate of 2. fn merkle_tree_psd(leaves: &[Vec>]) -> Result> { MerkleTree::new(&*TESTNET_POSEIDON_4, &*TESTNET_POSEIDON_2, leaves) diff --git a/ledger/store/src/block/mod.rs b/ledger/store/src/block/mod.rs index 0ae0113b88..638d71b59b 100644 --- a/ledger/store/src/block/mod.rs +++ b/ledger/store/src/block/mod.rs @@ -112,6 +112,12 @@ fn to_confirmed_transaction( } } +/// The prefix of the block tree cache file, used to recognize its format. +/// +/// Bump the trailing digits whenever the cached payload changes, so that a cache +/// file written by an older version is discarded instead of failing to decode. +pub(crate) const BLOCK_TREE_CACHE_PREFIX: &[u8; 12] = b"aleo.tree.01"; + pub(crate) fn block_tree_cache_path>(storage: &B) -> Option { #[cfg(feature = "rocks")] { @@ -1269,7 +1275,10 @@ impl> BlockStore { // the number of syscalls involved with disk writes. 1MiB should provide // a good balance between the CPU cache and maximum disk throughput. let mut writer = BufWriter::with_capacity(1024 * 1024, file); - bincode::serialize_into(&mut writer, &&*self.tree.read())?; + writer.write_all(BLOCK_TREE_CACHE_PREFIX)?; + // Note: only the contents of the tree are cached, not its hashers; the latter are + // deterministic, and recreating them is far cheaper than deserializing them. + bincode::serialize_into(&mut writer, &self.tree.read().to_state())?; writer.flush()?; // TODO(ljedrz): this operation can already take ~2.5s, so we may want // to perform chunking and parallel serialization. This may be useful diff --git a/ledger/store/src/helpers/rocksdb/block.rs b/ledger/store/src/helpers/rocksdb/block.rs index 38f687942b..96c52f1b82 100644 --- a/ledger/store/src/helpers/rocksdb/block.rs +++ b/ledger/store/src/helpers/rocksdb/block.rs @@ -18,7 +18,7 @@ use crate::{ ConfirmedTxType, TransactionStore, TransitionStore, - block::block_tree_cache_path, + block::{BLOCK_TREE_CACHE_PREFIX, block_tree_cache_path}, helpers::{ rocksdb::{ BlockMap, @@ -251,14 +251,26 @@ impl BlockStorage for BlockDB { bail!("Failed to determine the block tree cache path"); }; - if let Ok(serialized_tree) = fs::read(&path) { - debug!("Loading the cached block tree from {}", path.display()); - - // Deserialize a ready block tree. - let ret = bincode::deserialize(&serialized_tree).or_else(|e| { - tracing::error!("Failed to deserialize the block tree ({e}), constructing from scratch"); - construct_from_scratch(self) - }); + if let Ok(cached) = fs::read(&path) { + let ret = match cached.strip_prefix(BLOCK_TREE_CACHE_PREFIX) { + Some(serialized_state) => { + debug!("Loading the cached block tree from {}", path.display()); + + // Deserialize the contents of the block tree, and recreate it from them. + bincode::deserialize(serialized_state) + .map_err(anyhow::Error::from) + .and_then(N::merkle_tree_bhp_from_state) + .or_else(|e| { + tracing::error!("Failed to load the cached block tree ({e}), constructing from scratch"); + construct_from_scratch(self) + }) + } + // The cache was written by a version using a different format; discard it. + None => { + debug!("Discarding the outdated block tree cache at {}", path.display()); + construct_from_scratch(self) + } + }; // Ensure that an old cached tree is not reused. let _ = fs::remove_file(path); From 7d2b6a6a6349f88397158771cb0205fa32e0408f Mon Sep 17 00:00:00 2001 From: ljedrz Date: Fri, 31 Jul 2026 12:45:47 +0200 Subject: [PATCH 2/2] chore: update the lockfile Signed-off-by: ljedrz --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 5bf22ffbde..30a1ba2a3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3429,6 +3429,7 @@ name = "snarkvm-console-collections" version = "4.9.0" dependencies = [ "aleo-std", + "bincode", "criterion", "indexmap 2.14.0", "locktick",