From 2f403897fca8d890183a16fae2699efeddf8378a Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 13 May 2026 19:38:53 +0300 Subject: [PATCH 01/72] Initial db-sync removal --- Cargo.lock | 137 ++- ethexe/network/Cargo.toml | 6 +- ethexe/network/src/bitswap.rs | 317 ++++++ ethexe/network/src/db_sync/mod.rs | 1237 --------------------- ethexe/network/src/db_sync/requests.rs | 1101 ------------------ ethexe/network/src/db_sync/responses.rs | 552 --------- ethexe/network/src/gossipsub.rs | 6 +- ethexe/network/src/injected.rs | 8 +- ethexe/network/src/lib.rs | 172 +-- ethexe/network/src/peer_score.rs | 28 +- ethexe/network/src/utils.rs | 38 +- ethexe/network/src/validator/discovery.rs | 3 +- ethexe/network/src/validator/topic.rs | 6 +- ethexe/service/src/fast_sync.rs | 2 +- ethexe/service/src/lib.rs | 60 +- 15 files changed, 497 insertions(+), 3176 deletions(-) create mode 100644 ethexe/network/src/bitswap.rs delete mode 100644 ethexe/network/src/db_sync/mod.rs delete mode 100644 ethexe/network/src/db_sync/requests.rs delete mode 100644 ethexe/network/src/db_sync/responses.rs diff --git a/Cargo.lock b/Cargo.lock index afd7b900b34..f3509512cee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1931,6 +1931,32 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" +[[package]] +name = "beetswap" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a920844a4696d53e72190a8348f0048a6799647b3448d38bef90b85e0310df0" +dependencies = [ + "asynchronous-codec 0.7.0", + "blockstore", + "bytes", + "cid 0.11.3", + "fnv", + "futures-core", + "futures-timer", + "futures-util", + "libp2p-core 0.43.2", + "libp2p-identity", + "libp2p-swarm 0.47.0", + "multihash-codetable", + "quick-protobuf", + "smallvec", + "thiserror 2.0.17", + "tracing", + "unsigned-varint 0.8.0", + "web-time", +] + [[package]] name = "bimap" version = "0.6.3" @@ -2174,6 +2200,18 @@ dependencies = [ "piper", ] +[[package]] +name = "blockstore" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509096e88e431095763b3f5ee1e1cdb09212e4d5b2eccc91ddea965deefedb7d" +dependencies = [ + "cid 0.11.3", + "dashmap 6.1.0", + "multihash 0.19.5", + "thiserror 2.0.17", +] + [[package]] name = "blst" version = "0.3.16" @@ -2718,6 +2756,17 @@ dependencies = [ "unsigned-varint 0.7.2", ] +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash 0.19.5", + "unsigned-varint 0.8.0", +] + [[package]] name = "cipher" version = "0.2.5" @@ -3688,7 +3737,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.114", ] [[package]] @@ -5337,6 +5386,9 @@ dependencies = [ "assert_matches", "async-trait", "auto_impl", + "beetswap", + "blockstore", + "cid 0.11.3", "derive_more 2.1.1", "ethexe-common", "ethexe-db", @@ -5355,6 +5407,7 @@ dependencies = [ "lru 0.16.3", "metrics", "metrics-derive", + "multihash 0.19.5", "nonempty 0.12.0", "parity-scale-codec", "prometheus-client 0.23.1", @@ -9865,7 +9918,7 @@ dependencies = [ "libp2p-identity", "log", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "multistream-select", "once_cell", "parking_lot 0.12.5", @@ -9891,7 +9944,7 @@ dependencies = [ "futures-timer", "libp2p-identity", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "multistream-select", "parking_lot 0.12.5", "pin-project", @@ -10022,7 +10075,7 @@ dependencies = [ "ed25519-dalek", "hkdf", "k256", - "multihash 0.19.3", + "multihash 0.19.5", "quick-protobuf", "rand 0.8.5", "serde", @@ -10177,7 +10230,7 @@ dependencies = [ "libp2p-identity", "log", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "once_cell", "quick-protobuf", "rand 0.8.5", @@ -11376,7 +11429,7 @@ dependencies = [ "data-encoding", "libp2p-identity", "multibase", - "multihash 0.19.3", + "multihash 0.19.5", "percent-encoding", "serde", "static_assertions", @@ -11407,7 +11460,7 @@ dependencies = [ "blake3", "core2", "digest 0.10.7", - "multihash-derive", + "multihash-derive 0.8.1", "sha2 0.10.9", "sha3", "unsigned-varint 0.7.2", @@ -11424,7 +11477,7 @@ dependencies = [ "blake3", "core2", "digest 0.10.7", - "multihash-derive", + "multihash-derive 0.8.1", "sha2 0.10.9", "sha3", "unsigned-varint 0.7.2", @@ -11432,14 +11485,32 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.3" +version = "0.19.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" dependencies = [ - "core2", "unsigned-varint 0.8.0", ] +[[package]] +name = "multihash-codetable" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67996849749d25f1da9f238e8ace2ece8f9d6bdf3f9750aaf2ae7de3a5cad8ea" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "blake3", + "core2", + "digest 0.10.7", + "multihash-derive 0.9.3", + "ripemd", + "sha1", + "sha2 0.10.9", + "sha3", + "strobe-rs", +] + [[package]] name = "multihash-derive" version = "0.8.1" @@ -11454,6 +11525,29 @@ dependencies = [ "synstructure 0.12.6", ] +[[package]] +name = "multihash-derive" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4723acdce756db211a53f01b3419dbdf63cb48cef5df86260f55309364735fbf" +dependencies = [ + "multihash 0.19.5", + "multihash-derive-impl", +] + +[[package]] +name = "multihash-derive-impl" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f932556f78452e5604cef711349d337ec081a9aa3c96e67b3127c8f5df05d550" +dependencies = [ + "proc-macro-crate 3.4.0", + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure 0.13.2", +] + [[package]] name = "multimap" version = "0.8.3" @@ -15360,7 +15454,7 @@ dependencies = [ "libp2p 0.52.4", "linked_hash_set", "log", - "multihash 0.19.3", + "multihash 0.19.5", "parity-scale-codec", "prost 0.12.6", "prost-build 0.12.6", @@ -16009,7 +16103,7 @@ dependencies = [ "litep2p", "log", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "rand 0.8.5", "thiserror 1.0.69", "zeroize", @@ -18263,6 +18357,19 @@ dependencies = [ "serde", ] +[[package]] +name = "strobe-rs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98fe17535ea31344936cc58d29fec9b500b0452ddc4cc24c429c8a921a0e84e5" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "keccak", + "subtle 2.6.1", + "zeroize", +] + [[package]] name = "strsim" version = "0.8.0" @@ -19623,9 +19730,9 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "ucd-trie" diff --git a/ethexe/network/Cargo.toml b/ethexe/network/Cargo.toml index 1e38e768d0f..f9e18effc68 100644 --- a/ethexe/network/Cargo.toml +++ b/ethexe/network/Cargo.toml @@ -21,9 +21,13 @@ ethexe-service-utils.workspace = true ethexe-common.workspace = true gprimitives = { workspace = true, features = ["std", "codec"] } -# libp2p +# libp2p & friends libp2p = { version = "0.56.0", features = ["mdns", "gossipsub", "kad", "identify", "ping", "secp256k1", "request-response", "quic", "tcp", "dns", "tls", "tokio", "macros", "plaintext", "yamux", "metrics"] } libp2p-gossipsub = { version = "0.49.4", features = ["metrics"] } +beetswap = "0.5.0" +blockstore = "0.8.0" +cid = "0.11.3" +multihash = "0.19.5" # other deps tokio = { workspace = true, features = ["macros", "sync"] } diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs new file mode 100644 index 00000000000..f96302d3436 --- /dev/null +++ b/ethexe/network/src/bitswap.rs @@ -0,0 +1,317 @@ +// This file is part of Gear. +// +// Copyright (C) 2026 Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use beetswap::multihasher::{Multihasher, MultihasherError}; +use blockstore::{block::CidError, cond_send::CondSend}; +use cid::{Cid, CidGeneric}; +use ethexe_common::db::HashStorageRO; +use futures::FutureExt; +use gprimitives::H256; +use libp2p::{ + Multiaddr, PeerId, + core::{Endpoint, transport::PortUse}, + swarm::{ + ConnectionDenied, ConnectionId, FromSwarm, NetworkBehaviour, THandler, THandlerInEvent, + THandlerOutEvent, ToSwarm, + }, +}; +use multihash::Multihash; +use std::{ + collections::HashMap, + mem, + sync::Arc, + task::{Context, Poll}, +}; +use tokio::{ + sync::{mpsc, oneshot}, + task, +}; + +#[derive(Clone)] +pub struct Handle(mpsc::UnboundedSender<(H256, oneshot::Sender>)>); + +impl Handle { + pub async fn request(&self, request: H256) -> Vec { + let (tx, rx) = oneshot::channel(); + + self.0 + .send((request, tx)) + .expect("channel should never be closed"); + + rx.await.expect("channel should never be closed") + } +} + +pub(crate) trait BlockstoreDatabase: Send + Sync + HashStorageRO { + fn clone_boxed(&self) -> Box; +} + +impl BlockstoreDatabase for ethexe_db::Database { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } +} + +pub struct Blockstore { + db: Box, +} + +impl Blockstore { + const MAX_BLOCK_SIZE: u64 = 1024 * 1024; // 1MB + const BLAKE2B_CODE: u64 = 0xb220; + const CID_CODEC: u64 = 0x55; +} + +impl blockstore::Blockstore for Blockstore { + fn get( + &self, + cid: &CidGeneric, + ) -> impl Future>>> + CondSend { + let hash = *cid.hash(); + let db = self.db.clone_boxed(); + task::spawn_blocking(move || { + let hash: Multihash<32> = + beetswap::utils::convert_multihash(&hash).ok_or(blockstore::Error::CidTooLarge)?; + if hash.code() != Self::BLAKE2B_CODE { + return Err(blockstore::Error::CidError(CidError::InvalidMultihashCode( + hash.code(), + Self::BLAKE2B_CODE, + ))); + } + if hash.size() as usize != mem::size_of::() { + return Err(blockstore::Error::CidError( + CidError::InvalidMultihashLength(hash.size() as usize), + )); + } + + let hash = H256::from_slice(hash.digest()); + let data = db.read_by_hash(hash); + + if let Some(data) = &data + && data.len() as u64 > Self::MAX_BLOCK_SIZE + { + log::warn!("{hash} is too large: {} bytes", data.len()); + return Err(blockstore::Error::ValueTooLarge); + } + + Ok(data) + }) + .map(|res| res.expect("database panicked")) + } + + async fn put_keyed( + &self, + _cid: &CidGeneric, + _data: &[u8], + ) -> blockstore::Result<()> { + Ok(()) + } + + async fn remove(&self, _cid: &CidGeneric) -> blockstore::Result<()> { + Ok(()) + } + + async fn close(self) -> blockstore::Result<()> { + Ok(()) + } +} + +struct Blake2b256Multihasher; + +impl Multihasher<32> for Blake2b256Multihasher { + async fn hash( + &self, + multihash_code: u64, + input: &[u8], + ) -> Result, MultihasherError> { + if multihash_code != Blockstore::BLAKE2B_CODE { + return Err(MultihasherError::UnknownMultihashCode); + } + + let hash = ethexe_db::hash(input); + let hash = Multihash::wrap(Blockstore::BLAKE2B_CODE, hash.as_bytes()) + .expect("size is always correct"); + Ok(hash) + } +} + +type InnerBehaviour = beetswap::Behaviour<32, Blockstore>; + +pub struct Behaviour { + inner: InnerBehaviour, + handle: Handle, + rx: mpsc::UnboundedReceiver<(H256, oneshot::Sender>)>, + requests: HashMap>>, +} + +impl Behaviour { + pub fn new(db: Box) -> Self { + let (handle, rx) = mpsc::unbounded_channel(); + let blockstore = Arc::new(Blockstore { db }); + + Self { + inner: InnerBehaviour::builder(blockstore) + .register_multihasher(Blake2b256Multihasher) + .protocol_prefix("/ethexe") + .expect("prefix is always correct") + .build(), + handle: Handle(handle), + rx, + requests: HashMap::new(), + } + } + + pub fn handle(&self) -> Handle { + self.handle.clone() + } + + fn cid(hash: H256) -> Cid { + Cid::new_v1( + Blockstore::CID_CODEC, + Multihash::wrap(Blockstore::BLAKE2B_CODE, hash.as_bytes()) + .expect("size is always correct"), + ) + } + + fn handle_inner_event(&mut self, event: beetswap::Event) { + match event { + beetswap::Event::GetQueryResponse { query_id, data } => { + if let Some(channel) = self.requests.remove(&query_id) { + let _ = channel.send(data); + } + } + beetswap::Event::GetQueryError { query_id, error } => { + // The wrapper builds CIDs itself, so invalid multihashes are impossible. + // Blockstore errors here mean local storage violated its read contract. + panic!("{query_id:?} query failed: {error}"); + } + } + } +} + +impl NetworkBehaviour for Behaviour { + type ConnectionHandler = THandler; + type ToSwarm = (); + + fn handle_pending_inbound_connection( + &mut self, + connection_id: ConnectionId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ConnectionDenied> { + self.inner + .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) + } + + fn handle_established_inbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result, ConnectionDenied> { + self.inner.handle_established_inbound_connection( + connection_id, + peer, + local_addr, + remote_addr, + ) + } + + fn handle_pending_outbound_connection( + &mut self, + connection_id: ConnectionId, + maybe_peer: Option, + addresses: &[Multiaddr], + effective_role: Endpoint, + ) -> Result, ConnectionDenied> { + self.inner.handle_pending_outbound_connection( + connection_id, + maybe_peer, + addresses, + effective_role, + ) + } + + fn handle_established_outbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + addr: &Multiaddr, + role_override: Endpoint, + port_use: PortUse, + ) -> Result, ConnectionDenied> { + self.inner.handle_established_outbound_connection( + connection_id, + peer, + addr, + role_override, + port_use, + ) + } + + fn on_swarm_event(&mut self, event: FromSwarm) { + self.inner.on_swarm_event(event); + } + + fn on_connection_handler_event( + &mut self, + peer_id: PeerId, + connection_id: ConnectionId, + event: THandlerOutEvent, + ) { + self.inner + .on_connection_handler_event(peer_id, connection_id, event); + } + + fn poll( + &mut self, + cx: &mut Context<'_>, + ) -> Poll>> { + self.requests.retain(|&query_id, channel| { + if channel.is_closed() { + self.inner.cancel(query_id); + return false; + } + + true + }); + + while let Poll::Ready(Some((hash, channel))) = self.rx.poll_recv(cx) { + let cid = Self::cid(hash); + let query_id = self.inner.get(&cid); + self.requests.insert(query_id, channel); + } + + if let Poll::Ready(to_swarm) = self.inner.poll(cx) { + return match to_swarm { + ToSwarm::GenerateEvent(event) => { + self.handle_inner_event(event); + Poll::Pending + } + to_swarm => { + Poll::Ready(to_swarm.map_out(|_event| { + unreachable!("`ToSwarm::GenerateEvent` is handled above") + })) + } + }; + } + + Poll::Pending + } +} diff --git a/ethexe/network/src/db_sync/mod.rs b/ethexe/network/src/db_sync/mod.rs deleted file mode 100644 index a3771a4d82b..00000000000 --- a/ethexe/network/src/db_sync/mod.rs +++ /dev/null @@ -1,1237 +0,0 @@ -// This file is part of Gear. -// -// Copyright (C) 2024-2025 Gear Technologies Inc. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Peer-to-peer database synchronization for `ethexe`. -//! -//! The protocol is built on libp2p request/response and is used to fetch data -//! that can be revalidated locally: raw CAS blobs, program-to-code mappings, -//! valid code sets, and announce chains. Requests are driven through -//! [`Handle`], while the behaviour internally retries across peers, enforces a -//! per-request timeout, and limits concurrent inbound responses. - -mod requests; -mod responses; - -pub(crate) use crate::{ - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - db_sync::{requests::RetriableRequest, responses::OngoingResponses}, - export::{Multiaddr, PeerId}, - utils::ParityScaleCodec, -}; -use crate::{db_sync::requests::OngoingRequests, peer_score, utils::AlternateCollectionFmt}; -use async_trait::async_trait; -use ethexe_common::{ - Announce, - db::{ - AnnounceStorageRO, BlockMetaStorageRO, CodesStorageRO, ConfigStorageRO, GlobalsStorageRO, - HashStorageRO, - }, - gear::CodeState, - network::{AnnouncesRequest, AnnouncesResponse}, -}; -use ethexe_db::Database; -use futures::FutureExt; -use gprimitives::{ActorId, CodeId, H256}; -use libp2p::{ - StreamProtocol, - core::{Endpoint, transport::PortUse}, - request_response, - request_response::{InboundFailure, Message, OutboundFailure, ProtocolSupport}, - swarm::{ - ConnectionDenied, ConnectionId, FromSwarm, NetworkBehaviour, THandler, THandlerInEvent, - THandlerOutEvent, ToSwarm, - }, -}; -use parity_scale_codec::{Decode, Encode}; -use std::{ - collections::{BTreeMap, BTreeSet}, - num::NonZeroU32, - pin::Pin, - sync::atomic::{AtomicU64, Ordering}, - task::{Context, Poll}, - time::Duration, -}; -use tokio::sync::{mpsc, oneshot}; - -const STREAM_PROTOCOL: StreamProtocol = StreamProtocol::new("/ethexe/db-sync/1.0.0"); - -#[derive(Clone, metrics_derive::Metrics)] -#[metrics(scope = "ethexe_network_db_sync")] -struct Metrics { - /// Number of either active or pending requests - ongoing_requests: metrics::Gauge, - /// Number of incoming dropped requests - incoming_dropped_requests: metrics::Counter, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq, derive_more::Display)] -pub enum RequestFailure { - /// Request had been processing for too long - #[display("Request had been processing for too long")] - Timeout, -} - -#[derive(Debug, Eq, PartialEq)] -pub enum Event { - /// Request is in a pending state because there are no peers - NoPeers { - /// The ID of request - request_id: RequestId, - }, - /// Request completion done - RequestSucceed { - /// The ID of request - request_id: RequestId, - }, - /// Request failed - RequestFailed { - /// The failed request - request_id: RequestId, - /// Reason of request failure - error: RequestFailure, - }, - /// Request canceled - /// - /// User dropped [`HandleFuture`]. - /// - /// NOTE: `Event` is not guaranteed in a multithreaded environment - RequestCancelled { - /// The canceled request - request_id: RequestId, - }, - /// Incoming request - IncomingRequest { - /// The ID of in-progress response - response_id: ResponseId, - /// Peer who requested - peer_id: PeerId, - }, - /// Request dropped because simultaneous limit exceeded - IncomingRequestDropped { - /// Peer who should have received the response - peer_id: PeerId, - }, - /// Response sent to incoming request - ResponseSent { - /// The ID of completed response - response_id: ResponseId, - /// Peer who should receive response - peer_id: PeerId, - }, -} - -#[derive(Debug, Clone)] -pub(crate) struct Config { - pub request_timeout: Duration, - pub max_simultaneous_responses: u32, - pub max_chain_len_for_announces_response: NonZeroU32, -} - -impl Default for Config { - fn default() -> Self { - Self { - request_timeout: Duration::from_secs(100), - max_simultaneous_responses: 10, - max_chain_len_for_announces_response: DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - } - } -} - -#[cfg(test)] // used only in tests yet -impl Config { - pub(crate) fn with_request_timeout(mut self, request_timeout: Duration) -> Self { - self.request_timeout = request_timeout; - self - } - - pub(crate) fn with_max_simultaneous_responses( - mut self, - max_simultaneous_responses: u32, - ) -> Self { - self.max_simultaneous_responses = max_simultaneous_responses; - self - } -} - -/// An asynchronous provider of external blockchain data required for response validation. -#[async_trait] -pub trait ExternalDataProvider: Send + Sync { - /// Clone the provider as a trait object. - fn clone_boxed(&self) -> Box; - - /// Resolve program IDs to code IDs at the given block. - async fn programs_code_ids_at( - self: Box, - program_ids: BTreeSet, - block: H256, - ) -> anyhow::Result>; - - /// Resolve code IDs to code states at the given block. - async fn codes_states_at( - self: Box, - code_ids: BTreeSet, - block: H256, - ) -> anyhow::Result>; -} - -#[derive(Debug, Eq, PartialEq, Copy, Clone, Hash)] -pub struct RequestId(u64); - -impl RequestId { - fn next() -> Self { - static COUNTER: AtomicU64 = AtomicU64::new(0); - RequestId(COUNTER.fetch_add(1, Ordering::Relaxed)) - } -} - -#[derive(Debug, Eq, PartialEq, Copy, Clone, Hash)] -pub struct ResponseId(pub(crate) u64); - -#[derive(derive_more::Debug, Default, Clone, Eq, PartialEq, Encode, Decode, derive_more::From)] -pub struct HashesRequest( - #[debug("{:?}", AlternateCollectionFmt::set(_0, "hashes"))] pub BTreeSet, -); - -/// Request to fetch the program-to-code mapping visible at a specific block. -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct ProgramIdsRequest { - pub at: H256, - pub expected_count: u64, -} - -/// Request to fetch the current set of valid codes and verify the response -/// using [`ExternalDataProvider`] at a specific block. -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct ValidCodesRequest { - pub at: H256, - pub validated_count: u64, -} - -/// High-level db-sync request types supported by the network. -#[derive(Debug, Clone, Eq, PartialEq, derive_more::From)] -pub enum Request { - /// Fetch raw CAS blobs by hash. - Hashes(HashesRequest), - /// Fetch the program-to-code mapping for a block. - ProgramIds(ProgramIdsRequest), - /// Fetch the node's locally stored set of valid code IDs. - ValidCodes(ValidCodesRequest), - /// Fetch an announce chain segment. - Announces(AnnouncesRequest), -} - -impl Request { - /// Build a request for a set of CAS hashes. - pub fn hashes(request: impl Into>) -> Self { - Self::Hashes(HashesRequest(request.into())) - } - - /// Build a request for program-to-code mappings at `at`. - pub fn program_ids(at: H256, expected_count: u64) -> Self { - Self::ProgramIds(ProgramIdsRequest { at, expected_count }) - } - - /// Build a request for the valid code set, using `at` only for response - /// verification. - pub fn valid_codes(at: H256, validated_count: u64) -> Self { - Self::ValidCodes(ValidCodesRequest { - at, - validated_count, - }) - } -} - -/// Successful db-sync responses returned to callers. -#[derive(derive_more::Debug, Clone, Eq, PartialEq, derive_more::From, derive_more::Unwrap)] -pub enum Response { - /// Raw CAS blobs keyed by hash. - Hashes(#[debug("{:?}", AlternateCollectionFmt::map(_0, "entries"))] BTreeMap>), - /// Program-to-code mapping reconstructed for a block. - ProgramIds( - #[debug("{:?}", AlternateCollectionFmt::map(_0, "programs"))] BTreeMap, - ), - /// Set of valid code IDs known at a block. - ValidCodes(#[debug("{:?}", AlternateCollectionFmt::set(_0, "codes"))] BTreeSet), - /// Contiguous announce chain response. - Announces(AnnouncesResponse), -} - -/// Result delivered by [`HandleFuture`]. -pub type HandleResult = Result; - -enum HandleAction { - Request(RequestId, Request), - Retry(RetriableRequest), -} - -impl HandleAction { - fn request_id(&self) -> RequestId { - match self { - HandleAction::Request(request_id, _) => *request_id, - HandleAction::Retry(request) => request.id(), - } - } -} - -/// Future returned by [`Handle::request`] and [`Handle::retry`]. -pub struct HandleFuture { - request_id: RequestId, - rx: oneshot::Receiver, -} - -impl HandleFuture { - /// Returns the identifier assigned to this request. - pub fn request_id(&self) -> RequestId { - self.request_id - } -} - -impl Future for HandleFuture { - type Output = HandleResult; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.rx - .poll_unpin(cx) - .map(|res| res.expect("channel should never be closed")) - } -} - -#[derive(Clone)] -pub struct Handle(mpsc::UnboundedSender<(HandleAction, oneshot::Sender)>); - -impl Handle { - fn send(&self, action: HandleAction) -> HandleFuture { - let (tx, rx) = oneshot::channel(); - let request_id = action.request_id(); - - self.0 - .send((action, tx)) - .expect("channel should never be closed"); - - HandleFuture { request_id, rx } - } - - /// Enqueue a new request. - pub fn request(&self, request: Request) -> HandleFuture { - self.send(HandleAction::Request(RequestId::next(), request)) - } - - /// Re-enqueue a retriable request returned by a previous failure. - pub fn retry(&self, request: RetriableRequest) -> HandleFuture { - self.send(HandleAction::Retry(request)) - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode)] -pub(crate) struct InnerProgramIdsRequest { - at: H256, -} - -/// Network-only type to be encoded-decoded and sent over the network -#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode, derive_more::From)] -pub(crate) enum InnerRequest { - Hashes(HashesRequest), - ProgramIds(InnerProgramIdsRequest), - ValidCodes, - Announces(AnnouncesRequest), -} - -#[derive(Debug, Clone, Default, Eq, PartialEq, Encode, Decode)] -pub(crate) struct InnerHashesResponse(BTreeMap>); - -#[derive(Debug, Default, Eq, PartialEq, Encode, Decode)] -pub(crate) struct InnerProgramIdsResponse(BTreeSet); - -// TODO #4911: can be optimized - only not-base announces could be returned. -/// Response for announces request. -/// Must contain all announces for the requested range. -/// Must be sorted from predecessors to successors. -#[derive(Debug, Clone, Default, PartialEq, Eq, Encode, Decode)] -pub(crate) struct InnerAnnouncesResponse(Vec); - -/// Network-only type to be encoded-decoded and sent over the network -#[derive(Debug, Eq, PartialEq, derive_more::From, derive_more::Unwrap, Encode, Decode)] -pub(crate) enum InnerResponse { - Hashes(InnerHashesResponse), - ProgramIds(InnerProgramIdsResponse), - ValidCodes(BTreeSet), - Announces(InnerAnnouncesResponse), -} - -type InnerBehaviour = request_response::Behaviour>; - -#[auto_impl::auto_impl(&, Box)] -pub trait DbSyncDatabase: - Send - + HashStorageRO - + BlockMetaStorageRO - + AnnounceStorageRO - + CodesStorageRO - + ConfigStorageRO - + GlobalsStorageRO -{ - /// Clone the database as a trait object. - fn clone_boxed(&self) -> Box; -} - -impl DbSyncDatabase for Database { - fn clone_boxed(&self) -> Box { - Box::new(self.clone()) - } -} - -pub(crate) struct Behaviour { - inner: InnerBehaviour, - handle: Handle, - rx: mpsc::UnboundedReceiver<(HandleAction, oneshot::Sender)>, - ongoing_requests: OngoingRequests, - ongoing_responses: OngoingResponses, - metrics: Metrics, -} - -impl Behaviour { - pub(crate) fn new( - config: Config, - peer_score_handle: peer_score::Handle, - external_data_provider: Box, - db: Box, - ) -> Self { - let (handle, rx) = mpsc::unbounded_channel(); - let handle = Handle(handle); - - Self { - inner: InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ), - handle, - rx, - ongoing_requests: OngoingRequests::new( - &config, - peer_score_handle, - external_data_provider, - ), - ongoing_responses: OngoingResponses::new(db, &config), - metrics: Metrics::default(), - } - } - - pub fn handle(&self) -> Handle { - self.handle.clone() - } - - fn handle_inner_event( - &mut self, - event: request_response::Event, - ) -> Poll>> { - match event { - request_response::Event::Message { - peer, - connection_id: _, - message: - Message::Request { - request_id: _, - request, - channel, - }, - } => { - let response_id = self - .ongoing_responses - .handle_response(peer, channel, request); - - let event = if let Some(response_id) = response_id { - Event::IncomingRequest { - response_id, - peer_id: peer, - } - } else { - self.metrics.incoming_dropped_requests.increment(1); - Event::IncomingRequestDropped { peer_id: peer } - }; - - return Poll::Ready(ToSwarm::GenerateEvent(event)); - } - request_response::Event::Message { - peer: _, - connection_id: _, - message: - Message::Response { - request_id, - response, - }, - } => { - self.ongoing_requests.on_peer_response(request_id, response); - } - request_response::Event::OutboundFailure { - peer, - connection_id: _, - request_id, - error, - } => { - log::trace!("outbound failure for request {request_id} to {peer}: {error}"); - - if let OutboundFailure::UnsupportedProtocols = error { - log::debug!( - "request to {peer} failed because it doesn't support {STREAM_PROTOCOL} protocol" - ); - } - - self.ongoing_requests.on_peer_failure(request_id); - } - request_response::Event::InboundFailure { - peer, - connection_id: _, - request_id: _, - error: InboundFailure::UnsupportedProtocols, - } => { - log::debug!( - "request from {peer} failed because it doesn't support {STREAM_PROTOCOL} protocol" - ); - } - request_response::Event::InboundFailure { .. } => {} - request_response::Event::ResponseSent { .. } => {} - } - - Poll::Pending - } -} - -impl NetworkBehaviour for Behaviour { - type ConnectionHandler = THandler; - type ToSwarm = Event; - - fn handle_pending_inbound_connection( - &mut self, - connection_id: ConnectionId, - local_addr: &Multiaddr, - remote_addr: &Multiaddr, - ) -> Result<(), ConnectionDenied> { - self.inner - .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) - } - - fn handle_established_inbound_connection( - &mut self, - connection_id: ConnectionId, - peer: PeerId, - local_addr: &Multiaddr, - remote_addr: &Multiaddr, - ) -> Result, ConnectionDenied> { - self.inner.handle_established_inbound_connection( - connection_id, - peer, - local_addr, - remote_addr, - ) - } - - fn handle_pending_outbound_connection( - &mut self, - connection_id: ConnectionId, - maybe_peer: Option, - addresses: &[Multiaddr], - effective_role: Endpoint, - ) -> Result, ConnectionDenied> { - self.inner.handle_pending_outbound_connection( - connection_id, - maybe_peer, - addresses, - effective_role, - ) - } - - fn handle_established_outbound_connection( - &mut self, - connection_id: ConnectionId, - peer: PeerId, - addr: &Multiaddr, - role_override: Endpoint, - port_use: PortUse, - ) -> Result, ConnectionDenied> { - self.inner.handle_established_outbound_connection( - connection_id, - peer, - addr, - role_override, - port_use, - ) - } - - fn on_swarm_event(&mut self, event: FromSwarm) { - self.inner.on_swarm_event(event); - self.ongoing_requests.on_swarm_event(event); - } - - fn on_connection_handler_event( - &mut self, - peer_id: PeerId, - connection_id: ConnectionId, - event: THandlerOutEvent, - ) { - self.inner - .on_connection_handler_event(peer_id, connection_id, event) - } - - fn poll( - &mut self, - cx: &mut Context<'_>, - ) -> Poll>> { - if let Poll::Ready(Some((action, channel))) = self.rx.poll_recv(cx) { - match action { - HandleAction::Request(request_id, request) => { - self.ongoing_requests.request(request_id, request, channel); - } - HandleAction::Retry(request) => { - self.ongoing_requests.retry(request, channel); - } - } - } - - if let Poll::Ready(request_event) = - self.ongoing_requests - .poll(cx, &mut self.inner, &self.metrics) - { - return Poll::Ready(ToSwarm::GenerateEvent(request_event)); - } - - if let Poll::Ready((peer_id, response_id)) = - self.ongoing_responses.poll(cx, &mut self.inner) - { - return Poll::Ready(ToSwarm::GenerateEvent(Event::ResponseSent { - response_id, - peer_id, - })); - } - - if let Poll::Ready(to_swarm) = self.inner.poll(cx) { - return match to_swarm { - ToSwarm::GenerateEvent(event) => self.handle_inner_event(event), - to_swarm => Poll::Ready(to_swarm.map_out::(|_event| { - unreachable!("`ToSwarm::GenerateEvent` is handled above") - })), - }; - } - - Poll::Pending - } -} - -#[cfg(test)] -pub(crate) mod tests { - use super::*; - use crate::{tests::DataProvider, utils::tests::init_logger}; - use assert_matches::assert_matches; - use ethexe_common::{Announce, HashOf, StateHashWithQueueSize, db::*}; - use ethexe_db::Database; - use libp2p::{ - Swarm, Transport, - core::{transport::MemoryTransport, upgrade::Version}, - futures::StreamExt, - identity::Keypair, - swarm, - swarm::SwarmEvent, - }; - use libp2p_swarm_test::SwarmExt; - use std::{iter, mem}; - use tokio::time; - - // exactly like `Swarm::new_ephemeral_tokio` but we can pass our own config - fn new_ephemeral_swarm( - config: swarm::Config, - behaviour: T, - ) -> Swarm { - let identity = Keypair::generate_ed25519(); - let peer_id = PeerId::from(identity.public()); - - let transport = MemoryTransport::default() - .or_transport(libp2p::tcp::tokio::Transport::default()) - .upgrade(Version::V1) - .authenticate(libp2p::plaintext::Config::new(&identity)) - .multiplex(libp2p::yamux::Config::default()) - .timeout(Duration::from_secs(20)) - .boxed(); - - Swarm::new(transport, behaviour, peer_id, config) - } - - async fn new_swarm_with_config(config: Config) -> (Swarm, Database, DataProvider) { - let data_provider = DataProvider::default(); - let db = Database::memory(); - let behaviour = Behaviour::new( - config, - peer_score::Handle::new_test(), - data_provider.clone_boxed(), - Box::new(db.clone()), - ); - let mut swarm = Swarm::new_ephemeral_tokio(move |_keypair| behaviour); - swarm.listen().with_memory_addr_external().await; - (swarm, db, data_provider) - } - - async fn new_swarm() -> (Swarm, Database, DataProvider) { - new_swarm_with_config(Config::default()).await - } - - #[tokio::test] - async fn smoke() { - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let (mut bob, bob_db, _data_provider) = new_swarm().await; - - let hello_hash = bob_db.cas().write(b"hello"); - let world_hash = bob_db.cas().write(b"world"); - - alice.connect(&mut bob).await; - tokio::spawn(async move { - let mut values = None; - - while let Some(event) = bob.next().await { - let Ok(event) = event.try_into_behaviour_event() else { - continue; - }; - - match event { - Event::IncomingRequest { - response_id, - peer_id, - } => { - values = Some((response_id, peer_id)); - } - Event::ResponseSent { - response_id, - peer_id, - } => { - let (initial_response_id, initial_peer_id) = - values.expect("IncomingRequest must be first"); - assert_eq!(initial_response_id, response_id); - assert_eq!(initial_peer_id, peer_id); - } - _ => {} - } - } - }); - - let request = alice_handle.request(Request::hashes([hello_hash, world_hash])); - let request_id = request.request_id(); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes( - [ - (hello_hash, b"hello".to_vec()), - (world_hash, b"world".to_vec()) - ] - .into() - ) - ) - } - - #[tokio::test(start_paused = true)] - async fn timeout() { - init_logger(); - - let alice_config = Config::default().with_request_timeout(Duration::from_secs(3)); - let (mut alice, _alice_db, _data_provider) = new_swarm_with_config(alice_config).await; - let alice_handle = alice.behaviour().handle(); - - let mut bob = Swarm::new_ephemeral_tokio(|_keypair| { - InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ) - }); - bob.connect(&mut alice).await; - - let request = alice_handle.request(Request::hashes([])); - let request_id = request.request_id(); - - tokio::spawn(async move { - while let Some(event) = bob.next().await { - if let Ok(request_response::Event::Message { - message: - Message::Request { - channel, request, .. - }, - .. - }) = event.try_into_behaviour_event() - { - assert_eq!(request, InnerRequest::Hashes(HashesRequest::default())); - // just ignore request - mem::forget(channel); - } - } - }); - - time::advance(Config::default().request_timeout).await; - - let event = alice.next_behaviour_event().await; - assert_eq!( - event, - Event::RequestFailed { - request_id, - error: RequestFailure::Timeout, - } - ); - request.await.unwrap_err(); - } - - #[tokio::test] - async fn excessive_data_stripped() { - const DATA: [[u8; 1]; 3] = [*b"1", *b"2", *b"3"]; - - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - - let mut bob = Swarm::new_ephemeral_tokio(move |_keypair| { - InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ) - }); - bob.connect(&mut alice).await; - - let data_0 = ethexe_db::hash(&DATA[0]); - let data_1 = ethexe_db::hash(&DATA[1]); - let data_2 = ethexe_db::hash(&DATA[2]); - - let request = alice_handle.request(Request::hashes([data_0, data_1])); - let request_id = request.request_id(); - - tokio::spawn(async move { - while let Some(event) = bob.next().await { - if let Ok(request_response::Event::Message { - message: - Message::Request { - channel, request, .. - }, - .. - }) = event.try_into_behaviour_event() - { - assert_eq!( - request, - InnerRequest::Hashes(HashesRequest([data_0, data_1].into())) - ); - bob.behaviour_mut() - .send_response( - channel, - InnerHashesResponse( - [ - (data_0, DATA[0].to_vec()), - (data_1, DATA[1].to_vec()), - (data_2, DATA[2].to_vec()), - ] - .into(), - ) - .into(), - ) - .unwrap(); - } - } - }); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes([(data_0, DATA[0].to_vec()), (data_1, DATA[1].to_vec())].into()) - ); - } - - #[tokio::test] - async fn truncated_hashes_response_completed_from_same_peer() { - const DATA_LEN: usize = 6 * 1024 * 1024; - - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let (mut bob, bob_db, _data_provider) = new_swarm().await; - - let data_0 = vec![0; DATA_LEN]; - let data_1 = vec![1; DATA_LEN]; - let hash_0 = bob_db.cas().write(&data_0); - let hash_1 = bob_db.cas().write(&data_1); - - alice.connect(&mut bob).await; - tokio::spawn(bob.loop_on_next()); - - let request = alice_handle.request(Request::hashes([hash_0, hash_1])); - let request_id = request.request_id(); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes([(hash_0, data_0), (hash_1, data_1)].into()) - ); - } - - #[tokio::test] - async fn request_response_type_mismatch() { - init_logger(); - - let alice_config = Config::default().with_request_timeout(Duration::ZERO); - let (mut alice, _alice_db, _data_provider) = new_swarm_with_config(alice_config).await; - let alice_handle = alice.behaviour().handle(); - - let mut bob = Swarm::new_ephemeral_tokio(move |_keypair| { - InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ) - }); - bob.connect(&mut alice).await; - - let request = alice_handle.request(Request::hashes([])); - let request_id = request.request_id(); - - tokio::spawn(async move { - while let Some(event) = bob.next().await { - if let Ok(request_response::Event::Message { - message: - Message::Request { - channel, request, .. - }, - .. - }) = event.try_into_behaviour_event() - { - assert_eq!(request, InnerRequest::Hashes(HashesRequest::default())); - bob.behaviour_mut() - .send_response(channel, InnerProgramIdsResponse::default().into()) - .unwrap(); - } - } - }); - - let event = alice.next_behaviour_event().await; - assert_eq!( - event, - Event::RequestFailed { - request_id, - error: RequestFailure::Timeout, - } - ); - - request.await.unwrap_err(); - } - - #[tokio::test] - async fn request_completed_with_3_peers() { - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let (mut bob, bob_db, _data_provider) = new_swarm().await; - let (mut charlie, charlie_db, _data_provider) = new_swarm().await; - let (mut dave, dave_db, _data_provider) = new_swarm().await; - - alice.connect(&mut bob).await; - alice.connect(&mut charlie).await; - alice.connect(&mut dave).await; - tokio::spawn(bob.loop_on_next()); - tokio::spawn(charlie.loop_on_next()); - tokio::spawn(dave.loop_on_next()); - - let hello_hash = bob_db.cas().write(b"hello"); - let world_hash = charlie_db.cas().write(b"world"); - let mark_hash = dave_db.cas().write(b"!"); - - let request = alice_handle.request(Request::hashes([hello_hash, world_hash, mark_hash])); - let request_id = request.request_id(); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes( - [ - (hello_hash, b"hello".to_vec()), - (world_hash, b"world".to_vec()), - (mark_hash, b"!".to_vec()), - ] - .into() - ) - ); - } - - #[tokio::test] - async fn request_completed_after_new_peer() { - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let (mut bob, bob_db, _data_provider) = new_swarm().await; - let (charlie, charlie_db, _data_provider) = new_swarm().await; - let charlie_addr = charlie.external_addresses().next().cloned().unwrap(); - - alice.connect(&mut bob).await; - tokio::spawn(bob.loop_on_next()); - - let hello_hash = bob_db.cas().write(b"hello"); - let world_hash = charlie_db.cas().write(b"world"); - - let request = alice_handle.request(Request::hashes([hello_hash, world_hash])); - let request_id = request.request_id(); - - // first attempt - let event = alice.next_behaviour_event().now_or_never(); - assert_eq!(event, None); - - tokio::spawn(charlie.loop_on_next()); - alice.dial_and_wait(charlie_addr).await; - - // second attempt - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes( - [ - (hello_hash, b"hello".to_vec()), - (world_hash, b"world".to_vec()) - ] - .into() - ) - ); - } - - #[tokio::test(start_paused = true)] - async fn unsupported_protocol_handled() { - const REQUEST_TIMEOUT: Duration = Duration::from_secs(2); - - init_logger(); - - let alice_config = Config::default().with_request_timeout(REQUEST_TIMEOUT); - let (mut alice, _alice_db, _data_provider) = new_swarm_with_config(alice_config).await; - let alice_handle = alice.behaviour().handle(); - - // idle connection timeout is lowered because `libp2p` uses `future_timer` inside, - // so we cannot advance time like in tokio - let mut bob = new_ephemeral_swarm( - swarm::Config::with_tokio_executor() - .with_idle_connection_timeout(Duration::from_secs(5)), - InnerBehaviour::new([], request_response::Config::default()), - ); - let bob_peer_id = *bob.local_peer_id(); - bob.connect(&mut alice).await; - tokio::spawn(bob.loop_on_next()); - - let request = alice_handle.request(Request::hashes([])); - let request_id = request.request_id(); - - // activate timer - let event = alice.next_behaviour_event().now_or_never(); - assert_eq!(event, None); - - time::advance(REQUEST_TIMEOUT).await; - - let event = alice.next_behaviour_event().await; - assert_eq!( - event, - Event::RequestFailed { - request_id, - error: RequestFailure::Timeout - } - ); - - let event = alice.next_swarm_event().await; - assert_matches!(event, SwarmEvent::ConnectionClosed { peer_id, .. } if peer_id == bob_peer_id); - } - - #[tokio::test] - async fn simultaneous_responses_limit() { - init_logger(); - - let alice_config = Config::default().with_max_simultaneous_responses(0); - let (mut alice, _alice_db, _data_provider) = new_swarm_with_config(alice_config).await; - - let (mut bob, _bob_db, _data_provider) = new_swarm().await; - let bob_handle = bob.behaviour().handle(); - let bob_peer_id = *bob.local_peer_id(); - - alice.connect(&mut bob).await; - tokio::spawn(bob.loop_on_next()); - - let fut = bob_handle.request(Request::hashes([])); - mem::forget(fut); - - let event = alice.next_behaviour_event().await; - assert_matches!(event, Event::IncomingRequestDropped { peer_id } if peer_id == bob_peer_id); - } - - #[tokio::test(start_paused = true)] - async fn retry() { - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let mut bob = Swarm::new_ephemeral_tokio(move |_keypair| { - InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ) - }); - bob.connect(&mut alice).await; - - let request_key = ethexe_db::hash(b"test"); - let request = alice_handle.request(Request::hashes([request_key])); - let request_id = request.request_id(); - - let bob_handle = tokio::spawn(async move { - while let Some(event) = bob.next().await { - if let Ok(request_response::Event::Message { - message: - Message::Request { - channel, request, .. - }, - .. - }) = event.try_into_behaviour_event() - { - assert_eq!( - request, - InnerRequest::Hashes(HashesRequest([request_key].into())) - ); - // just ignore request - mem::forget(channel); - } - } - }); - - time::advance(Config::default().request_timeout).await; - - // first attempt - let event = alice.next_behaviour_event().await; - assert_eq!( - event, - Event::RequestFailed { - request_id, - error: RequestFailure::Timeout, - } - ); - let (error, retriable_request) = request.await.unwrap_err(); - assert_eq!(error, RequestFailure::Timeout); - - time::resume(); - - bob_handle.abort(); - assert!(bob_handle.await.unwrap_err().is_cancelled()); - let (mut charlie, charlie_db, _data_provider) = new_swarm().await; - alice.connect(&mut charlie).await; - tokio::spawn(charlie.loop_on_next()); - - let key = charlie_db.cas().write(b"test"); - assert_eq!(request_key, key); - let request = alice_handle.retry(retriable_request); - let request_id = request.request_id(); - - // retry attempt - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes([(request_key, b"test".to_vec())].into()) - ); - } - - #[tokio::test] - async fn external_data_provider() { - init_logger(); - - let (mut alice, _alice_db, alice_data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let (mut bob, bob_db, _data_provider) = new_swarm().await; - - let expected_response = fill_data_provider(alice_data_provider, bob_db).await; - - alice.connect(&mut bob).await; - tokio::spawn(bob.loop_on_next()); - - let request = alice_handle.request(Request::program_ids(H256::zero(), 2)); - let request_id = request.request_id(); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!(response, expected_response); - } - - #[tokio::test] - async fn request_cancelled() { - let (mut alice, _db, _data_provider) = new_swarm().await; - - let request = alice.behaviour().handle().request(Request::hashes([])); - let request_id = request.request_id(); - drop(request); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestCancelled { request_id }); - } - - pub(crate) async fn fill_data_provider( - // data provider of the first peer - left_data_provider: DataProvider, - // database of the second peer - right_db: Database, - ) -> Response { - let program_ids: BTreeSet = [ActorId::new([1; 32]), ActorId::new([2; 32])].into(); - let code_ids = vec![CodeId::new([0xfe; 32]), CodeId::new([0xef; 32])]; - left_data_provider - .set_programs_code_ids_at(program_ids.clone(), H256::zero(), code_ids.clone()) - .await; - - let announce = Announce::base(H256::zero(), HashOf::zero()); - let announce_hash = announce.to_hash(); - right_db.mutate_block_announces(H256::zero(), |announces| { - announces.insert(announce_hash); - }); - - right_db.set_announce_program_states( - announce_hash, - iter::zip( - program_ids.clone(), - iter::repeat_with(H256::random).map(|hash| StateHashWithQueueSize { - hash, - canonical_queue_size: 0, - injected_queue_size: 0, - }), - ) - .collect(), - ); - - Response::ProgramIds(iter::zip(program_ids, code_ids).collect()) - } -} diff --git a/ethexe/network/src/db_sync/requests.rs b/ethexe/network/src/db_sync/requests.rs deleted file mode 100644 index f393569fc1d..00000000000 --- a/ethexe/network/src/db_sync/requests.rs +++ /dev/null @@ -1,1101 +0,0 @@ -// This file is part of Gear. -// -// Copyright (C) 2025 Gear Technologies Inc. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use crate::{ - db_sync::{ - AnnouncesRequest, Config, Event, ExternalDataProvider, HandleResult, HashesRequest, - InnerAnnouncesResponse, InnerBehaviour, InnerHashesResponse, InnerProgramIdsRequest, - InnerProgramIdsResponse, InnerRequest, InnerResponse, Metrics, PeerId, ProgramIdsRequest, - Request, RequestFailure, RequestId, Response, ValidCodesRequest, - }, - peer_score::Handle, - utils::{ConnectionMap, NoLimits}, -}; -use anyhow::Context as _; -use ethexe_common::{ - Announce, HashOf, - gear::CodeState, - network::{AnnouncesRequestUntil, AnnouncesResponse}, -}; -use futures::{FutureExt, future::BoxFuture}; -use gprimitives::{ActorId, CodeId, H256}; -use itertools::EitherOrBoth; -use libp2p::{request_response::OutboundRequestId, swarm::FromSwarm}; -use rand::prelude::IteratorRandom; -use std::{ - cell::OnceCell, - collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}, - iter, - task::{Context, Poll, Waker}, - time::Duration, -}; -use tokio::{sync::oneshot, time}; - -ethexe_service_utils::task_local! { - static CONTEXT: OngoingRequestContext; -} - -type OngoingRequestFuture = BoxFuture<'static, Result>; - -pub(crate) struct OngoingRequests { - pending_events: VecDeque, - requests: HashMap>)>, - active_requests: HashMap, - responses: HashMap>, - connections: ConnectionMap, - waker: Option, - // used in requests themselves - peer_score_handle: Handle, - external_data_provider: Box, - // config - request_timeout: Duration, -} - -impl OngoingRequests { - pub(crate) fn new( - config: &Config, - peer_score_handle: Handle, - external_data_provider: Box, - ) -> Self { - Self { - pending_events: VecDeque::new(), - requests: Default::default(), - active_requests: Default::default(), - responses: Default::default(), - connections: ConnectionMap::without_limits(), - waker: None, - peer_score_handle, - external_data_provider, - request_timeout: config.request_timeout, - } - } - - fn wake(&mut self) { - if let Some(waker) = self.waker.take() { - waker.wake(); - } - } - - /// Tracks all active connections. - pub(crate) fn on_swarm_event(&mut self, event: FromSwarm) { - if self.connections.on_swarm_event(event) { - self.wake(); - } - } - - fn inner_request( - &mut self, - request_id: RequestId, - request: OngoingRequest, - channel: oneshot::Sender, - ) { - self.requests.insert( - request_id, - ( - request - .request( - self.peer_score_handle.clone(), - self.external_data_provider.clone_boxed(), - self.request_timeout, - ) - .boxed(), - Some(channel), - ), - ); - } - - pub(crate) fn request( - &mut self, - request_id: RequestId, - request: Request, - channel: oneshot::Sender, - ) { - self.inner_request(request_id, OngoingRequest::new(request), channel); - } - - pub(crate) fn retry( - &mut self, - request: RetriableRequest, - channel: oneshot::Sender, - ) { - let RetriableRequest { - request_id, - request, - } = request; - self.inner_request(request_id, request, channel); - } - - fn inner_on_peer( - &mut self, - outbound_request_id: OutboundRequestId, - res: Result, - ) { - let request_id = self - .active_requests - .remove(&outbound_request_id) - .expect("unknown outbound request id"); - let fut = self.requests.get_mut(&request_id); - - // request can be removed because of timeout, - // so we don't expect it's still inside `self.requests` - if fut.is_some() { - self.responses.insert(request_id, res); - self.wake(); - } else { - log::trace!("{outbound_request_id:?} has been skipped for {request_id:?}"); - } - } - - pub(crate) fn on_peer_response( - &mut self, - outbound_request_id: OutboundRequestId, - response: InnerResponse, - ) { - self.inner_on_peer(outbound_request_id, Ok(response)); - } - - pub(crate) fn on_peer_failure(&mut self, outbound_request_id: OutboundRequestId) { - self.inner_on_peer(outbound_request_id, Err(())); - } - - pub(crate) fn poll( - &mut self, - cx: &mut Context<'_>, - behaviour: &mut InnerBehaviour, - metrics: &Metrics, - ) -> Poll { - loop { - if let Some(event) = self.pending_events.pop_front() { - return Poll::Ready(event); - } - - let peers: HashSet = self.connections.peers().collect(); - - self.requests.retain(|&request_id, (fut, channel)| { - let response = self.responses.remove(&request_id); - - // it means `HandleFuture` is dropped, - // so we just remove the request and don't make any further work - if channel.as_ref().expect("always Some").is_closed() { - self.pending_events.push_back(Event::RequestCancelled { request_id }); - return false; - } - - let ctx = OngoingRequestContext { - state: OnceCell::new(), - peers: peers.clone(), - response, - }; - - let (ctx, poll) = CONTEXT.scope(ctx, || fut.poll_unpin(cx)); - let state = ctx.into_state(); - if state.is_some() && poll.is_ready() { - unreachable!( - "state machine invariant violated: unexpected ready poll with existing state" - ); - } - - if let Some(state) = state { - match state { - OngoingRequestState::NoPeers => { - - self.pending_events.push_back(Event::NoPeers { request_id }); - }, - OngoingRequestState::SendRequest(peer, request, ) => { - let outbound_request_id = behaviour.send_request(&peer, request); - self.active_requests.insert(outbound_request_id, request_id); - } - }; - } else if let Poll::Ready(res) = poll { - let (event, res) = match res { - Ok(response) => { - (Event::RequestSucceed { request_id }, Ok(response)) - } - Err((error, request)) => { - (Event::RequestFailed { request_id, error }, Err((error, RetriableRequest { - request_id, - request, - }, - ))) - } - }; - - self.pending_events.push_back(event); - - // channel can be dropped after `is_closed()` check during future polling - let res = channel.take().expect("always Some").send(res); - if res.is_err() { - self.pending_events.push_back(Event::RequestCancelled { request_id }); - } - - return false; - } - - true - }); - metrics.ongoing_requests.set(self.requests.len() as f64); - - // it means some futures are pending, so we definitely will wake the task - if !self.requests.is_empty() { - self.waker = Some(cx.waker().clone()); - } - - if !self.pending_events.is_empty() { - // immediately return event instead of task waking - continue; - } - - break Poll::Pending; - } - } -} - -#[derive(Debug)] -enum HashesResponseHandled { - Done { - response: BTreeMap>, - stripped: bool, - }, - IncompleteData { - acc: InnerHashesResponse, - new_request: HashesRequest, - stripped: bool, - }, - Err { - acc: InnerHashesResponse, - err: HashesResponseError, - stripped: bool, - }, -} - -impl HashesResponseHandled { - fn stripped(&self) -> bool { - match self { - Self::Done { stripped, .. } => *stripped, - Self::IncompleteData { stripped, .. } => *stripped, - Self::Err { stripped, .. } => *stripped, - } - } -} - -#[derive(Debug, derive_more::Unwrap)] -pub(crate) enum AnnouncesResponseHandled { - Done(AnnouncesResponse), - NewRound, - Err(AnnouncesResponseError), -} - -#[derive(Debug, Copy, Clone, Eq, PartialEq, derive_more::Display)] -pub enum HashesResponseError { - #[display("hash mismatch from provided data")] - HashMismatch, -} - -#[derive(Debug, derive_more::Display)] -pub enum ProgramIdsResponseError { - #[display("not enough program-code ids expected {expected} but got {received}")] - NotEnoughIds { expected: usize, received: usize }, - #[display("router failed: {_0}")] - RouterQuery(anyhow::Error), -} - -#[derive(Debug, derive_more::Display)] -pub enum ValidCodesResponseError { - #[display("not enough validated codes")] - NotEnoughCodes, - #[display("{_0}")] - RouterQuery(anyhow::Error), -} - -#[derive(Debug, PartialEq, Eq, derive_more::Display)] -pub enum AnnouncesResponseError { - #[display("announces head mismatch, expected hash {expected}, received {received}")] - HeadMismatch { - expected: HashOf, - received: HashOf, - }, - #[display("announces tail mismatch, expected hash {expected}, received {received}")] - TailMismatch { - expected: HashOf, - received: HashOf, - }, - #[display("announces len expected {expected}, received {received}")] - LenMismatch { expected: usize, received: usize }, - #[display("announces chain is not linked")] - ChainIsNotLinked, -} - -#[derive(Debug, derive_more::Display, derive_more::From)] -pub(crate) enum ResponseError { - #[display("{_0}")] - Hashes(HashesResponseError), - #[display("{_0}")] - ProgramIds(ProgramIdsResponseError), - #[display("{_0}")] - ValidCodes(ValidCodesResponseError), - #[display("{_0}")] - Announces(AnnouncesResponseError), - #[display("request and response types mismatch")] - TypeMismatch, -} - -#[derive(Debug, derive_more::Unwrap)] -pub(crate) enum ResponseHandlerResult { - Ok(Response), - NewRound(ResponseHandler), - Err(ResponseHandler, ResponseError), -} - -impl From> for ResponseHandlerResult { - fn from(value: Result) -> Self { - match value { - Ok(response) => Self::Ok(response), - Err((handler, error)) => Self::Err(handler, error), - } - } -} - -#[derive(Debug, Eq, PartialEq)] -pub(crate) enum ResponseHandler { - Hashes { - acc: InnerHashesResponse, - request: HashesRequest, - }, - ProgramIds { - request: ProgramIdsRequest, - }, - ValidCodes { - request: ValidCodesRequest, - }, - Announces { - request: AnnouncesRequest, - }, -} - -impl ResponseHandler { - fn new(request: Request) -> Self { - match request { - Request::Hashes(request) => Self::Hashes { - acc: Default::default(), - request, - }, - Request::ProgramIds(request) => Self::ProgramIds { request }, - Request::ValidCodes(request) => Self::ValidCodes { request }, - Request::Announces(request) => Self::Announces { request }, - } - } - - fn inner_request(&self) -> InnerRequest { - match self { - ResponseHandler::Hashes { - request: reduced_request, - .. - } => InnerRequest::Hashes(reduced_request.clone()), - ResponseHandler::ProgramIds { - request: - ProgramIdsRequest { - at, - expected_count: _, - }, - } => InnerRequest::ProgramIds(InnerProgramIdsRequest { at: *at }), - ResponseHandler::ValidCodes { - request: - ValidCodesRequest { - at: _, - validated_count: _, - }, - } => InnerRequest::ValidCodes, - ResponseHandler::Announces { request } => InnerRequest::Announces(*request), - } - } - - fn handle_hashes( - mut acc: InnerHashesResponse, - reduced_request: &HashesRequest, - new_response: InnerHashesResponse, - ) -> HashesResponseHandled { - let mut new_request = BTreeSet::new(); - let mut stripped = false; - - let diff = itertools::merge_join_by( - reduced_request.0.iter().copied(), - new_response.0, - |req_key, (resp_key, _resp_val)| req_key.cmp(resp_key), - ); - - for either in diff { - match either { - EitherOrBoth::Both(req_key, (resp_key, resp_val)) => { - debug_assert_eq!(req_key, resp_key); - if req_key != ethexe_db::hash(&resp_val) { - return HashesResponseHandled::Err { - acc, - err: HashesResponseError::HashMismatch, - stripped, - }; - } - - acc.0.insert(resp_key, resp_val); - } - EitherOrBoth::Left(key) => { - // peer was unable to give this key - new_request.insert(key); - } - EitherOrBoth::Right(_key) => { - // peer sent more keys than we requested - stripped = true; - } - } - } - - if new_request.is_empty() { - HashesResponseHandled::Done { - response: acc.0, - stripped, - } - } else { - HashesResponseHandled::IncompleteData { - acc, - new_request: HashesRequest(new_request), - stripped, - } - } - } - - async fn handle_program_ids( - response: InnerProgramIdsResponse, - request: &ProgramIdsRequest, - external_data_provider: Box, - ) -> Result, ProgramIdsResponseError> { - let InnerProgramIdsResponse(response) = response; - - if response.len() as u64 != request.expected_count { - return Err(ProgramIdsResponseError::NotEnoughIds { - expected: request.expected_count as usize, - received: response.len(), - }); - } - - let code_ids = external_data_provider - .programs_code_ids_at(response.clone(), request.at) - .await - .context("failed to get code ids at block") - .map_err(ProgramIdsResponseError::RouterQuery)?; - - let program_code_ids = iter::zip(response, code_ids).collect(); - Ok(program_code_ids) - } - - async fn handle_valid_codes( - response: BTreeSet, - request: &ValidCodesRequest, - external_data_provider: Box, - ) -> Result, ValidCodesResponseError> { - // validated count at specified block can be less than - // the number of states at the latest block returned by peer - // but cannot be more - if (response.len() as u64) < request.validated_count { - return Err(ValidCodesResponseError::NotEnoughCodes); - } - - let states = external_data_provider - .codes_states_at(response.clone(), request.at) - .await - .context("failed to get code states at block") - .map_err(ValidCodesResponseError::RouterQuery)?; - - let code_ids: BTreeSet = iter::zip(response, states) - .flat_map(|(code_id, state)| { - if state == CodeState::Validated { - Some(code_id) - } else { - None - } - }) - .collect(); - if request.validated_count != code_ids.len() as u64 { - return Err(ValidCodesResponseError::NotEnoughCodes); - } - - Ok(code_ids) - } - - pub(crate) fn handle_announces( - response: InnerAnnouncesResponse, - request: AnnouncesRequest, - ) -> AnnouncesResponseHandled { - let InnerAnnouncesResponse(announces) = response; - - let Some((first, last)) = announces.first().zip(announces.last()) else { - return AnnouncesResponseHandled::NewRound; - }; - - if request.head != last.to_hash() { - return AnnouncesResponseHandled::Err(AnnouncesResponseError::HeadMismatch { - expected: request.head, - received: last.to_hash(), - }); - } - - match request.until { - AnnouncesRequestUntil::Tail(request_tail_hash) => { - if request_tail_hash != first.parent { - return AnnouncesResponseHandled::Err(AnnouncesResponseError::TailMismatch { - expected: request_tail_hash, - received: first.parent, - }); - } - } - AnnouncesRequestUntil::ChainLen(len) => { - if announces.len() != len.get() as usize { - return AnnouncesResponseHandled::Err(AnnouncesResponseError::LenMismatch { - expected: len.get() as usize, - received: announces.len(), - }); - } - } - } - - // Check chain linking - let mut expected_parent_hash = first.parent; - for announce in announces.iter() { - if announce.parent != expected_parent_hash { - return AnnouncesResponseHandled::Err(AnnouncesResponseError::ChainIsNotLinked); - } - expected_parent_hash = announce.to_hash(); - } - - unsafe { AnnouncesResponseHandled::Done(AnnouncesResponse::from_parts(request, announces)) } - } - - async fn handle( - self, - peer: PeerId, - response: InnerResponse, - peer_score_handle: &Handle, - external_data_provider: Box, - ) -> ResponseHandlerResult { - match (self, response) { - ( - Self::Hashes { - acc, - request: reduced_request, - }, - InnerResponse::Hashes(response), - ) => { - let handled = Self::handle_hashes(acc, &reduced_request, response); - - if handled.stripped() { - log::debug!("data stripped in response from {peer}"); - peer_score_handle.excessive_data(peer); - } - - match handled { - HashesResponseHandled::Done { - response, - stripped: _, - } => ResponseHandlerResult::Ok(Response::Hashes(response)), - HashesResponseHandled::IncompleteData { - acc, - new_request, - stripped: _, - } => ResponseHandlerResult::NewRound(Self::Hashes { - acc, - request: new_request, - }), - HashesResponseHandled::Err { - acc, - err, - stripped: _, - } => ResponseHandlerResult::Err( - Self::Hashes { - acc, - request: reduced_request, - }, - err.into(), - ), - } - } - (Self::ProgramIds { request }, InnerResponse::ProgramIds(response)) => { - Self::handle_program_ids(response, &request, external_data_provider) - .await - .map(Into::into) - .map_err(|err| (Self::ProgramIds { request }, err.into())) - .into() - } - (Self::ValidCodes { request }, InnerResponse::ValidCodes(response)) => { - Self::handle_valid_codes(response, &request, external_data_provider) - .await - .map(Into::into) - .map_err(|err| (Self::ValidCodes { request }, err.into())) - .into() - } - (Self::Announces { request }, InnerResponse::Announces(response)) => { - let handled = Self::handle_announces(response, request); - - match handled { - AnnouncesResponseHandled::Done(response) => { - ResponseHandlerResult::Ok(Response::Announces(response)) - } - AnnouncesResponseHandled::NewRound => { - ResponseHandlerResult::NewRound(Self::Announces { request }) - } - AnnouncesResponseHandled::Err(err) => { - ResponseHandlerResult::Err(Self::Announces { request }, err.into()) - } - } - } - (this, _) => ResponseHandlerResult::Err(this, ResponseError::TypeMismatch), - } - } -} - -#[derive(Debug)] -struct OngoingRequest { - response_handler: Option, - tried_peers: HashSet, -} - -impl OngoingRequest { - fn new(request: Request) -> Self { - Self { - response_handler: Some(ResponseHandler::new(request)), - tried_peers: Default::default(), - } - } - - async fn choose_next_peer(&mut self) -> PeerId { - let mut event_sent = None; - CONTEXT - .poll_fn(|_task_cx, ctx| { - if ctx.peers.is_empty() { - event_sent.get_or_insert_with(|| { - ctx.state - .set(OngoingRequestState::NoPeers) - .expect("set only once"); - }); - return Poll::Pending; - } - - loop { - let peer = ctx - .peers - .difference(&self.tried_peers) - .choose_stable(&mut rand::thread_rng()) - .copied(); - - if let Some(peer) = peer { - self.tried_peers.insert(peer); - break Poll::Ready(peer); - } else { - // just retry all peers again - self.tried_peers.clear(); - continue; - } - } - }) - .await - } - - async fn send_request(&mut self, peer: PeerId) -> Result { - CONTEXT.with_mut(|ctx| { - ctx.state - .set(OngoingRequestState::SendRequest( - peer, - self.response_handler - .as_ref() - .expect("always Some") - .inner_request(), - )) - .expect("set only once"); - }); - - CONTEXT - .poll_fn(|_task_cx, ctx| { - if let Some(res) = ctx.response.take() { - Poll::Ready(res) - } else { - Poll::Pending - } - }) - .await - } - - async fn next_round( - &mut self, - peer_score_handle: &Handle, - external_data_provider: Box, - ) -> Result { - let peer = self.choose_next_peer().await; - - let response = self.send_request(peer).await?; - - match self - .response_handler - .take() - .expect("always Some") - .handle(peer, response, peer_score_handle, external_data_provider) - .await - { - ResponseHandlerResult::Ok(response) => Ok(response), - ResponseHandlerResult::NewRound(handler) => { - log::trace!( - "response is incomplete from peer {peer}: we are going for a new round" - ); - self.response_handler = Some(handler); - Err(()) - } - ResponseHandlerResult::Err(handler, err) => { - log::warn!("response processing failed for request from {peer}: {err:?}"); - peer_score_handle.invalid_data(peer); - self.response_handler = Some(handler); - Err(()) - } - } - } - - async fn request( - mut self, - peer_score_handle: Handle, - external_data_provider: Box, - request_timeout: Duration, - ) -> Result { - let request_loop = async { - loop { - match self - .next_round(&peer_score_handle, external_data_provider.clone_boxed()) - .await - { - Ok(response) => break Ok(response), - Err(()) => continue, - }; - } - }; - - let res = time::timeout(request_timeout, request_loop) - .await - .map_err(|_elapsed| RequestFailure::Timeout) - .and_then(|res| res); - res.map_err(|failure| (failure, self)) - } -} - -#[derive(Debug)] -enum OngoingRequestState { - NoPeers, - SendRequest(PeerId, InnerRequest), -} - -struct OngoingRequestContext { - state: OnceCell, - peers: HashSet, - response: Option>, -} - -impl OngoingRequestContext { - fn into_state(self) -> Option { - let Self { - state, - peers: _, - response, - } = self; - let state = state.into_inner(); - debug_assert_eq!(response, None, "future must take provided response"); - state - } -} - -#[derive(Debug)] -pub struct RetriableRequest { - request_id: RequestId, - request: OngoingRequest, -} - -impl PartialEq for RetriableRequest { - fn eq(&self, other: &Self) -> bool { - self.request_id == other.request_id - } -} - -impl Eq for RetriableRequest {} - -impl RetriableRequest { - pub fn id(&self) -> RequestId { - self.request_id - } -} - -#[cfg(test)] -mod tests { - use super::*; - use async_trait::async_trait; - - struct UnreachableExternalDataProvider; - - #[async_trait] - impl ExternalDataProvider for UnreachableExternalDataProvider { - fn clone_boxed(&self) -> Box { - unreachable!() - } - - async fn programs_code_ids_at( - self: Box, - _program_ids: BTreeSet, - _block: H256, - ) -> anyhow::Result> { - unreachable!() - } - - async fn codes_states_at( - self: Box, - _code_ids: BTreeSet, - _block: H256, - ) -> anyhow::Result> { - unreachable!() - } - } - - fn make_chain(len: usize) -> Vec { - assert!(len > 0); - let mut chain = Vec::with_capacity(len); - let mut parent = HashOf::zero(); - - for idx in 0..len { - let announce = Announce::base(H256([idx as u8 + 1; 32]), parent); - parent = announce.to_hash(); - chain.push(announce); - } - - chain - } - - #[test] - fn validate_data_stripped() { - let hash1 = ethexe_db::hash(b"1"); - let hash2 = ethexe_db::hash(b"2"); - let hash3 = ethexe_db::hash(b"3"); - - let request = HashesRequest([hash1, hash2].into()); - let response = InnerHashesResponse( - [ - (hash1, b"1".to_vec()), - (hash2, b"2".to_vec()), - (hash3, b"3".to_vec()), - ] - .into(), - ); - let processed = ResponseHandler::handle_hashes(Default::default(), &request, response); - let HashesResponseHandled::Done { response, stripped } = processed else { - unreachable!("{processed:?}") - }; - assert_eq!( - response, - BTreeMap::from_iter([(hash1, b"1".to_vec()), (hash2, b"2".to_vec())]) - ); - assert!(stripped); - } - - #[test] - fn validate_data_hash_mismatch() { - let hash1 = ethexe_db::hash(b"1"); - - let request = HashesRequest([hash1].into()); - let response = InnerHashesResponse([(hash1, b"2".to_vec())].into()); - let processed = ResponseHandler::handle_hashes(Default::default(), &request, response); - let HashesResponseHandled::Err { acc, err, stripped } = processed else { - unreachable!("{processed:?}") - }; - assert_eq!(acc, Default::default()); - assert_eq!(err, HashesResponseError::HashMismatch); - assert!(!stripped); - } - - #[tokio::test] - async fn validate_data_hash_incomplete() { - let hash1 = ethexe_db::hash(b"1"); - let hash2 = ethexe_db::hash(b"2"); - - let request = HashesRequest([hash1, hash2].into()); - let response = InnerHashesResponse([(hash1, b"1".to_vec())].into()); - let processed = - ResponseHandler::handle_hashes(Default::default(), &request, response.clone()); - let HashesResponseHandled::IncompleteData { - acc, - new_request, - stripped, - } = processed - else { - unreachable!("{processed:?}") - }; - assert_eq!(acc, response); - assert_eq!(new_request, HashesRequest([hash2].into())); - assert!(!stripped); - - let handler = ResponseHandler::new(request.into()); - handler - .handle( - PeerId::random(), - response.into(), - &Handle::new_test(), - Box::new(UnreachableExternalDataProvider), - ) - .await - .unwrap_new_round(); - } - - #[test] - fn try_into_checked_accepts_valid_tail_range() { - let announces = make_chain(3); - let head_hash = announces.last().unwrap().to_hash(); - let tail_hash = announces.first().unwrap().parent; - - let request = AnnouncesRequest { - head: head_hash, - until: AnnouncesRequestUntil::Tail(tail_hash), - }; - let response = InnerAnnouncesResponse(announces.clone()); - - let response = ResponseHandler::handle_announces(response, request).unwrap_done(); - assert_eq!(response.request(), &request); - assert_eq!(response.announces(), announces.as_slice()); - } - - #[test] - fn try_into_checked_accepts_valid_chain_len() { - let announces = make_chain(4); - let head_hash = announces.last().unwrap().to_hash(); - - let request = AnnouncesRequest { - head: head_hash, - until: AnnouncesRequestUntil::ChainLen((announces.len() as u32).try_into().unwrap()), - }; - - let response = InnerAnnouncesResponse(announces.clone()); - - let response = ResponseHandler::handle_announces(response, request).unwrap_done(); - assert_eq!(response.request(), &request); - assert_eq!(response.announces(), announces.as_slice()); - } - - #[tokio::test] - async fn try_into_checked_rejects_empty_response() { - let request = AnnouncesRequest { - head: HashOf::zero(), - until: AnnouncesRequestUntil::ChainLen(1.try_into().unwrap()), - }; - - let response = InnerAnnouncesResponse(Vec::new()); - - ResponseHandler::handle_announces(response.clone(), request).unwrap_new_round(); - - let handler = ResponseHandler::new(request.into()); - handler - .handle( - PeerId::random(), - response.into(), - &Handle::new_test(), - Box::new(UnreachableExternalDataProvider), - ) - .await - .unwrap_new_round(); - } - - #[test] - fn try_into_checked_rejects_head_mismatch() { - let announces = make_chain(2); - let actual_head = announces.last().unwrap().to_hash(); - let wrong_head = HashOf::random(); - let tail_hash = announces.first().unwrap().parent; - - let request = AnnouncesRequest { - head: wrong_head, - until: AnnouncesRequestUntil::Tail(tail_hash), - }; - let response = InnerAnnouncesResponse(announces); - - let err = ResponseHandler::handle_announces(response, request).unwrap_err(); - assert_eq!( - err, - AnnouncesResponseError::HeadMismatch { - expected: wrong_head, - received: actual_head, - } - ); - } - - #[test] - fn try_into_checked_rejects_tail_mismatch() { - let announces = make_chain(3); - let actual_tail = announces.first().unwrap().parent; - let head_hash = announces.last().unwrap().to_hash(); - let wrong_tail = HashOf::random(); - - let request = AnnouncesRequest { - head: head_hash, - until: AnnouncesRequestUntil::Tail(wrong_tail), - }; - let response = InnerAnnouncesResponse(announces); - - let err = ResponseHandler::handle_announces(response, request).unwrap_err(); - assert_eq!( - err, - AnnouncesResponseError::TailMismatch { - expected: wrong_tail, - received: actual_tail, - } - ); - } - - #[test] - fn try_into_checked_rejects_len_mismatch() { - let announces = make_chain(2); - let head_hash = announces.last().unwrap().to_hash(); - - let request = AnnouncesRequest { - head: head_hash, - until: AnnouncesRequestUntil::ChainLen(3.try_into().unwrap()), - }; - let response = InnerAnnouncesResponse(announces); - - let err = ResponseHandler::handle_announces(response, request).unwrap_err(); - assert_eq!( - err, - AnnouncesResponseError::LenMismatch { - expected: 3, - received: 2, - } - ); - } - - #[test] - fn try_into_checked_rejects_non_linked_chain() { - let mut announces = make_chain(3); - announces[1].parent = HashOf::zero(); - let head_hash = announces.last().unwrap().to_hash(); - let tail_hash = announces.first().unwrap().parent; - - let request = AnnouncesRequest { - head: head_hash, - until: AnnouncesRequestUntil::Tail(tail_hash), - }; - let response = InnerAnnouncesResponse(announces); - - let err = ResponseHandler::handle_announces(response, request).unwrap_err(); - assert_eq!(err, AnnouncesResponseError::ChainIsNotLinked); - } -} diff --git a/ethexe/network/src/db_sync/responses.rs b/ethexe/network/src/db_sync/responses.rs deleted file mode 100644 index b79d44bbb61..00000000000 --- a/ethexe/network/src/db_sync/responses.rs +++ /dev/null @@ -1,552 +0,0 @@ -// This file is part of Gear. -// -// Copyright (C) 2024-2025 Gear Technologies Inc. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use crate::{ - db_sync::{ - Config, DbSyncDatabase, InnerAnnouncesResponse, InnerBehaviour, InnerHashesResponse, - InnerProgramIdsResponse, InnerRequest, InnerResponse, ResponseId, - }, - export::PeerId, - utils::ParityScaleCodec, -}; -use ethexe_common::{ - Announce, HashOf, - db::{AnnounceStorageRO, ConfigStorageRO, GlobalsStorageRO}, - network::{AnnouncesRequest, AnnouncesRequestUntil}, -}; -use libp2p::request_response; -use parity_scale_codec::{Compact, Encode}; -use std::{ - collections::{BTreeMap, VecDeque}, - num::NonZeroU32, - task::{Context, Poll}, -}; -use thiserror::Error; -use tokio::task::JoinSet; - -struct OngoingResponse { - response_id: ResponseId, - peer_id: PeerId, - channel: request_response::ResponseChannel, - response: InnerResponse, -} - -pub(crate) struct OngoingResponses { - response_id_counter: u64, - db: Box, - db_readers: JoinSet, - max_simultaneous_responses: u32, - max_chain_len_for_announces_response: NonZeroU32, -} - -impl OngoingResponses { - pub(crate) fn new(db: Box, config: &Config) -> Self { - Self { - response_id_counter: 0, - db, - db_readers: JoinSet::new(), - max_simultaneous_responses: config.max_simultaneous_responses, - max_chain_len_for_announces_response: config.max_chain_len_for_announces_response, - } - } - - fn next_response_id(&mut self) -> ResponseId { - let id = self.response_id_counter; - self.response_id_counter += 1; - ResponseId(id) - } - - fn response_from_db( - request: InnerRequest, - db: Box, - max_chain_len_for_announces_response: NonZeroU32, - ) -> InnerResponse { - const MAX_RESPONSE_SIZE: u64 = ParityScaleCodec::<(), ()>::MAX_RESPONSE_SIZE; - - match request { - InnerRequest::Hashes(request) => { - let mut response = BTreeMap::new(); - let mut entries_size = 0; - - for hash in request.0 { - let Some(data) = db.read_by_hash(hash) else { - continue; - }; - - let entry_size = hash.encoded_size() + data.encoded_size(); - let next_response_size = 1 // InnerResponse discriminant size - + Compact((response.len() + 1) as u64).encoded_size() - + entries_size - + entry_size; - - if next_response_size > MAX_RESPONSE_SIZE as usize { - // don't try to put other hashes data to prevent abusive database reads - break; - } - - entries_size += entry_size; - response.insert(hash, data); - } - - InnerHashesResponse(response).into() - } - InnerRequest::ProgramIds(request) => InnerProgramIdsResponse( - db.block_announces(request.at) - .into_iter() - .flatten() - .find_map(|announce_hash| db.announce_program_states(announce_hash)) - .map(|states| states.into_keys().collect()) - .unwrap_or_else(|| { - log::warn!("no program states found for block {:?}", request.at); - Default::default() - }), // FIXME: Option might be more suitable - ) - .into(), - InnerRequest::ValidCodes => db.valid_codes().into(), - InnerRequest::Announces(request) => { - match Self::process_announce_request( - &db, - request, - max_chain_len_for_announces_response, - ) { - Ok(response) => response.into(), - Err(e) => { - log::trace!("cannot complete announces request {request:?}: {e}"); - InnerResponse::Announces(Default::default()) - } - } - } - } - } - - fn process_announce_request( - db: &DB, - request: AnnouncesRequest, - max_chain_len_for_announces_response: NonZeroU32, - ) -> Result { - let AnnouncesRequest { head, until } = request; - - // Check the requested chain length first to prevent abuse - if let AnnouncesRequestUntil::ChainLen(len) = until - && len > max_chain_len_for_announces_response - { - // TODO #4874: use peer score to punish the peer for such requests - return Err(ProcessAnnounceError::ChainLenExceedsMax { - requested: len, - max_allowed: max_chain_len_for_announces_response, - }); - } - - let genesis_announce_hash = db.config().genesis_announce_hash; - let start_announce_hash = db.globals().start_announce_hash; - - let mut announces = VecDeque::new(); - let mut announce_hash = head; - for _ in 0..max_chain_len_for_announces_response.get() { - match until { - AnnouncesRequestUntil::Tail(tail) if announce_hash == tail => { - return Ok(InnerAnnouncesResponse(announces.into())); - } - AnnouncesRequestUntil::ChainLen(len) if announces.len() == len.get() as usize => { - return Ok(InnerAnnouncesResponse(announces.into())); - } - _ => {} - } - - if announce_hash == start_announce_hash { - if start_announce_hash == genesis_announce_hash { - // Reaching genesis - request is invalid and should be punished. - // TODO #4874: use peer score to punish the peer for such requests - return Err(ProcessAnnounceError::ReachedGenesis { - genesis: genesis_announce_hash, - }); - } else { - // Reaching start announce - request can be valid, we just can't go further - return Err(ProcessAnnounceError::ReachedStart { - start: start_announce_hash, - }); - } - } - - let Some(announce) = db.announce(announce_hash) else { - return Err(ProcessAnnounceError::AnnounceMissing { - hash: announce_hash, - }); - }; - announce_hash = announce.parent; - announces.push_front(announce); - } - - // TODO #4874: use peer score to punish the peer for such requests - Err(ProcessAnnounceError::ReachedMaxChainLength { - max_allowed: max_chain_len_for_announces_response, - }) - } - - pub(crate) fn handle_response( - &mut self, - peer_id: PeerId, - channel: request_response::ResponseChannel, - request: InnerRequest, - ) -> Option { - if self.db_readers.len() >= self.max_simultaneous_responses as usize { - return None; - } - - let response_id = self.next_response_id(); - - let db = self.db.clone_boxed(); - let max_chain_len_for_announces_response = self.max_chain_len_for_announces_response; - self.db_readers.spawn_blocking(move || { - let response = - Self::response_from_db(request, db, max_chain_len_for_announces_response); - OngoingResponse { - response_id, - peer_id, - channel, - response, - } - }); - - Some(response_id) - } - - pub(crate) fn poll( - &mut self, - cx: &mut Context<'_>, - behaviour: &mut InnerBehaviour, - ) -> Poll<(PeerId, ResponseId)> { - if let Poll::Ready(Some(res)) = self.db_readers.poll_join_next(cx) { - let OngoingResponse { - response_id, - peer_id, - channel, - response, - } = res.expect("database panicked"); - let _res = behaviour.send_response(channel, response); - Poll::Ready((peer_id, response_id)) - } else { - Poll::Pending - } - } -} - -#[derive(Debug, Error, PartialEq, Eq)] -enum ProcessAnnounceError { - #[error("requested chain length {requested} exceeds maximum allowed {max_allowed}")] - ChainLenExceedsMax { - requested: NonZeroU32, - max_allowed: NonZeroU32, - }, - #[error("announce {hash} not found in database")] - AnnounceMissing { hash: HashOf }, - #[error("reached genesis announce {genesis}")] - ReachedGenesis { genesis: HashOf }, - #[error("reached start announce {start}")] - ReachedStart { start: HashOf }, - #[error("reached maximum chain length {max_allowed}")] - ReachedMaxChainLength { max_allowed: NonZeroU32 }, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - db_sync::{HashesRequest, requests::ResponseHandler}, - }; - use ethexe_common::{ - Announce, HashOf, ProtocolTimelines, - db::{AnnounceStorageRW, DBConfig, GlobalsStorageRW, SetConfig}, - }; - use ethexe_db::Database; - use gprimitives::H256; - use std::num::{NonZeroU32, NonZeroU64}; - - fn make_announce(block: u64, parent: HashOf) -> Announce { - Announce::base(H256::from_low_u64_be(block), parent) - } - - fn set_db_data(db: &Database, genesis: HashOf, start: HashOf) { - db.set_config(DBConfig { - version: 0, - chain_id: 0, - router_address: Default::default(), - timelines: ProtocolTimelines { - genesis_ts: 0, - era: NonZeroU64::new(1).unwrap(), - election: 0, - slot: NonZeroU64::new(1).unwrap(), - }, - genesis_block_hash: H256::zero(), - genesis_announce_hash: genesis, - max_validators: 0, - }); - - db.globals_mutate(|globals| globals.start_announce_hash = start); - } - - #[test] - fn response_from_db_truncates_hashes_response_at_encoded_limit() { - const ENTRIES_BEFORE_COMPACT_BOUNDARY: u64 = 0b0011_1111; - const MAX_RESPONSE_SIZE: usize = ParityScaleCodec::<(), ()>::MAX_RESPONSE_SIZE as usize; - - let db = Database::memory(); - - let entries = (0..ENTRIES_BEFORE_COMPACT_BOUNDARY as u8) - .map(|i| vec![i]) - .collect::>(); - let entries_size = entries - .iter() - .map(|data| H256::zero().encoded_size() + data.encoded_size()) - .sum::(); - for data in &entries { - db.cas().write(data); - } - - let last_entry_size = MAX_RESPONSE_SIZE - - 1 // `InnerResponse` discriminant - - Compact(ENTRIES_BEFORE_COMPACT_BOUNDARY + 1).encoded_size() - - entries_size - - H256::zero().encoded_size(); - let last_entry = vec![42; last_entry_size]; - let last_entry_hash = db.cas().write(&last_entry); - - let request = entries - .iter() - .map(|data| ethexe_db::hash(data)) - .chain(Some(last_entry_hash)) - .collect(); - let response = OngoingResponses::response_from_db( - HashesRequest(request).into(), - Box::new(db), - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - ); - - let response = response.unwrap_hashes(); - assert_eq!(response.0.len(), ENTRIES_BEFORE_COMPACT_BOUNDARY as usize); - assert!(InnerResponse::Hashes(response).encoded_size() <= MAX_RESPONSE_SIZE); - } - - #[test] - fn fails_chain_len_exceeding_max() { - let db = Database::memory(); - set_db_data(&db, HashOf::zero(), HashOf::zero()); - - let len = DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE - .checked_add(1) - .unwrap(); - let request = AnnouncesRequest { - head: HashOf::zero(), - until: AnnouncesRequestUntil::ChainLen(len), - }; - - let err = OngoingResponses::process_announce_request( - &db, - request, - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - ) - .unwrap_err(); - assert_eq!( - err, - ProcessAnnounceError::ChainLenExceedsMax { - requested: len, - max_allowed: DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - } - ); - } - - #[test] - fn fails_announce_missing() { - let head = HashOf::random(); - let db = Database::memory(); - set_db_data(&db, HashOf::zero(), HashOf::zero()); - - let request = AnnouncesRequest { - head, - until: AnnouncesRequestUntil::Tail(HashOf::zero()), - }; - - let err = OngoingResponses::process_announce_request( - &db, - request, - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - ) - .unwrap_err(); - assert_eq!(err, ProcessAnnounceError::AnnounceMissing { hash: head }); - } - - #[test] - fn fails_when_reaching_genesis() { - let db = Database::memory(); - - let genesis_announce = make_announce(10, HashOf::random()); - let genesis = db.set_announce(genesis_announce); - let middle = make_announce(11, genesis); - let middle_hash = db.set_announce(middle.clone()); - let head = make_announce(12, middle_hash); - let head_hash = db.set_announce(head.clone()); - - set_db_data(&db, genesis, genesis); - - let request = AnnouncesRequest { - head: head_hash, - until: AnnouncesRequestUntil::Tail(HashOf::random()), - }; - - let err = OngoingResponses::process_announce_request( - &db, - request, - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - ) - .unwrap_err(); - assert_eq!(err, ProcessAnnounceError::ReachedGenesis { genesis }); - } - - #[test] - fn fails_reaching_start_non_genesis() { - let db = Database::memory(); - let start_announce = make_announce(10, HashOf::random()); - let start = db.set_announce(start_announce); - let genesis = HashOf::random(); - - set_db_data(&db, genesis, start); - - let head = make_announce(11, start); - let head_hash = db.set_announce(head); - - let request = AnnouncesRequest { - head: head_hash, - until: AnnouncesRequestUntil::Tail(HashOf::random()), - }; - - let err = OngoingResponses::process_announce_request( - &db, - request, - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - ) - .unwrap_err(); - assert_eq!(err, ProcessAnnounceError::ReachedStart { start }); - } - - #[test] - fn fails_reaching_max_chain_length() { - let db = Database::memory(); - - let mut parent = HashOf::random(); - let mut head_hash = parent; - let mut chain_hashes = Vec::new(); - - for i in 0..DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE.get() { - let announce = make_announce(10_000 + i as u64, parent); - let hash = db.set_announce(announce); - chain_hashes.push(hash); - parent = hash; - head_hash = hash; - } - - let start = HashOf::random(); - let genesis = HashOf::random(); - let tail = HashOf::random(); - - assert!(!chain_hashes.contains(&start)); - assert!(!chain_hashes.contains(&genesis)); - assert!(!chain_hashes.contains(&tail)); - - set_db_data(&db, genesis, start); - - let request = AnnouncesRequest { - head: head_hash, - until: AnnouncesRequestUntil::Tail(tail), - }; - - let err = OngoingResponses::process_announce_request( - &db, - request, - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - ) - .unwrap_err(); - assert_eq!( - err, - ProcessAnnounceError::ReachedMaxChainLength { - max_allowed: DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - } - ); - } - - #[test] - fn returns_announces_until_tail() { - let db = Database::memory(); - - let tail = make_announce(10, HashOf::random()); - let tail_hash = db.set_announce(tail.clone()); - let middle = make_announce(11, tail_hash); - let middle_hash = db.set_announce(middle.clone()); - let head = make_announce(12, middle_hash); - let head_hash = db.set_announce(head.clone()); - - let genesis = HashOf::random(); - let start = HashOf::random(); - set_db_data(&db, genesis, start); - - let request = AnnouncesRequest { - head: head_hash, - until: AnnouncesRequestUntil::Tail(tail_hash), - }; - - let response = OngoingResponses::process_announce_request( - &db, - request, - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - ) - .unwrap(); - assert_eq!(response.0, vec![middle, head]); - ResponseHandler::handle_announces(response, request).unwrap_done(); - } - - #[test] - fn returns_announces_until_chain_len() { - let db = Database::memory(); - - let tail = make_announce(10, HashOf::random()); - let tail_hash = db.set_announce(tail.clone()); - let middle = make_announce(11, tail_hash); - let middle_hash = db.set_announce(middle.clone()); - let head = make_announce(12, middle_hash); - let head_hash = db.set_announce(head.clone()); - - let genesis = HashOf::random(); - let start = HashOf::random(); - set_db_data(&db, genesis, start); - - let length = NonZeroU32::new(2).unwrap(); - let request = AnnouncesRequest { - head: head_hash, - until: AnnouncesRequestUntil::ChainLen(length), - }; - - let response = OngoingResponses::process_announce_request( - &db, - request, - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - ) - .unwrap(); - assert_eq!(response.0, vec![middle, head]); - ResponseHandler::handle_announces(response, request).unwrap_done(); - } -} diff --git a/ethexe/network/src/gossipsub.rs b/ethexe/network/src/gossipsub.rs index cf1ee3e5bbc..7ab1e45209a 100644 --- a/ethexe/network/src/gossipsub.rs +++ b/ethexe/network/src/gossipsub.rs @@ -18,13 +18,11 @@ pub(crate) use libp2p::gossipsub::*; -use crate::{ - db_sync::{Multiaddr, PeerId}, - peer_score, -}; +use crate::peer_score; use anyhow::anyhow; use ethexe_common::{Address, injected::SignedCompactPromise, network::SignedValidatorMessage}; use libp2p::{ + Multiaddr, PeerId, core::{Endpoint, transport::PortUse}, gossipsub, identity::Keypair, diff --git a/ethexe/network/src/injected.rs b/ethexe/network/src/injected.rs index 0acd18c7c66..6f4777886fb 100644 --- a/ethexe/network/src/injected.rs +++ b/ethexe/network/src/injected.rs @@ -16,11 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::{ - db_sync::{Multiaddr, PeerId}, - utils::ParityScaleCodec, - validator::discovery::ValidatorIdentities, -}; +use crate::{utils::ParityScaleCodec, validator::discovery::ValidatorIdentities}; use ethexe_common::{ Address, HashOf, injected::{ @@ -30,7 +26,7 @@ use ethexe_common::{ }; use futures::{FutureExt, StreamExt, future::BoxFuture, stream::FuturesUnordered}; use libp2p::{ - StreamProtocol, + Multiaddr, PeerId, StreamProtocol, core::{Endpoint, transport::PortUse}, request_response, request_response::{ diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index ecf41d69aa3..a39146f2c93 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -33,7 +33,7 @@ //! hands out protocol-specific handles such as [`db_sync::Handle`] for //! database synchronization and a peer-scoring handle for internal use. -pub mod db_sync; +mod bitswap; mod gossipsub; mod injected; mod kad; @@ -50,7 +50,6 @@ pub mod export { pub use injected::Event as NetworkInjectedEvent; use crate::{ - db_sync::DbSyncDatabase, utils::MultiaddrExt, validator::{ValidatorDatabase, list::ValidatorListSnapshot}, }; @@ -201,8 +200,6 @@ pub struct NetworkRuntimeConfig { pub general_signer: Signer, /// Signer used only to construct the libp2p networking keypair. pub network_signer: Signer, - /// External lookups needed by db-sync request validation. - pub external_data_provider: Box, /// Database backing validator discovery and db-sync responses. pub db: Database, } @@ -276,7 +273,6 @@ impl NetworkService { validator_key, general_signer, network_signer, - external_data_provider, db, } = runtime_config; @@ -299,8 +295,7 @@ impl NetworkService { let behaviour_config = BehaviourConfig { router_address, keypair: keypair.clone(), - external_data_provider, - db: DbSyncDatabase::clone_boxed(&db), + db: bitswap::BlockstoreDatabase::clone_boxed(&db), transport_type, validator_key, general_signer, @@ -451,7 +446,7 @@ impl NetworkService { } BehaviourEvent::Kad(event) => self.handle_kad_event(event), BehaviourEvent::Gossipsub(event) => return self.handle_gossipsub_event(event), - BehaviourEvent::DbSync(_event) => {} + BehaviourEvent::Bitswap(_event) => {} BehaviourEvent::Injected(event) => return self.handle_injected_event(event), BehaviourEvent::ValidatorDiscovery(event) => { return self.handle_validator_discovery_event(event); @@ -630,9 +625,9 @@ impl NetworkService { self.swarm.behaviour().peer_score.handle() } - /// Handle used by external services to start db-sync requests. - pub fn db_sync_handle(&self) -> db_sync::Handle { - self.swarm.behaviour().db_sync.handle() + /// Handle used by external services to start Bitswap requests. + pub fn bitswap_handle(&self) -> bitswap::Handle { + self.swarm.behaviour().bitswap.handle() } /// Refresh validator-era state after the chain head changes. @@ -702,8 +697,7 @@ impl NetworkService { struct BehaviourConfig<'a> { router_address: Address, keypair: identity::Keypair, - external_data_provider: Box, - db: Box, + db: Box, transport_type: TransportType, validator_key: Option, general_signer: Signer, @@ -734,8 +728,8 @@ pub(crate) struct Behaviour { pub kad: kad::Behaviour, // general communication pub gossipsub: gossipsub::Behaviour, - // database synchronization protocol - pub db_sync: db_sync::Behaviour, + // data request protocol + pub bitswap: bitswap::Behaviour, // injected transaction shenanigans pub injected: injected::Behaviour, // validator discovery @@ -747,7 +741,6 @@ impl Behaviour { let BehaviourConfig { router_address, keypair, - external_data_provider, db, transport_type, validator_key, @@ -805,15 +798,7 @@ impl Behaviour { ) .map_err(|e| anyhow!("`gossipsub::Behaviour` error: {e}"))?; - let db_sync = db_sync::Behaviour::new( - db_sync::Config { - max_chain_len_for_announces_response, - ..Default::default() - }, - peer_score_handle.clone(), - external_data_provider, - db, - ); + let bitswap = bitswap::Behaviour::new(db); let injected = injected::Behaviour::new(); @@ -836,7 +821,7 @@ impl Behaviour { mdns4, kad, gossipsub, - db_sync, + bitswap, injected, validator_discovery, }) @@ -846,95 +831,22 @@ impl Behaviour { #[cfg(test)] mod tests { use super::*; - use crate::{ - db_sync::{ExternalDataProvider, tests::fill_data_provider}, - utils::tests::{arb_value, init_logger}, - }; + use crate::utils::tests::{arb_value, init_logger}; use assert_matches::assert_matches; - use async_trait::async_trait; - use ethexe_common::{BlockHeader, ProtocolTimelines, db::*, gear::CodeState}; + use ethexe_common::{BlockHeader, ProtocolTimelines, db::*}; use ethexe_db::Database; - use gprimitives::{ActorId, CodeId, H256}; + use futures::future; + use gprimitives::H256; use gsigner::secp256k1::Signer; use nonempty::nonempty; - use std::{ - collections::{BTreeSet, HashMap}, - future, - num::NonZeroU64, - sync::Arc, - }; + use std::num::NonZeroU64; use tokio::{ - sync::RwLock, time, time::{Duration, timeout}, }; - #[derive(Default)] - struct DataProviderInner { - programs_code_ids_at: HashMap<(BTreeSet, H256), Vec>, - code_states_at: HashMap<(BTreeSet, H256), Vec>, - } - - #[derive(Default, Clone)] - pub struct DataProvider(Arc>); - - impl DataProvider { - pub async fn set_programs_code_ids_at( - &self, - program_ids: BTreeSet, - at: H256, - code_ids: Vec, - ) { - self.0 - .write() - .await - .programs_code_ids_at - .insert((program_ids, at), code_ids); - } - } - - #[async_trait] - impl ExternalDataProvider for DataProvider { - fn clone_boxed(&self) -> Box { - Box::new(self.clone()) - } - - async fn programs_code_ids_at( - self: Box, - program_ids: BTreeSet, - block: H256, - ) -> anyhow::Result> { - assert!(!program_ids.is_empty()); - Ok(self - .0 - .read() - .await - .programs_code_ids_at - .get(&(program_ids, block)) - .cloned() - .unwrap_or_default()) - } - - async fn codes_states_at( - self: Box, - code_ids: BTreeSet, - block: H256, - ) -> anyhow::Result> { - assert!(!code_ids.is_empty()); - Ok(self - .0 - .read() - .await - .code_states_at - .get(&(code_ids, block)) - .cloned() - .unwrap_or_default()) - } - } - struct NetworkServiceBuilder { db: Database, - data_provider: DataProvider, latest_validators: ValidatorsVec, signer: Signer, validator_key: Option, @@ -944,7 +856,6 @@ mod tests { fn new() -> Self { Self { db: Database::memory(), - data_provider: DataProvider::default(), latest_validators: nonempty![Address::default()].into(), signer: Signer::memory(), validator_key: None, @@ -966,7 +877,6 @@ mod tests { let Self { db, - data_provider, latest_validators, signer, validator_key, @@ -986,7 +896,6 @@ mod tests { validator_key, general_signer: signer.clone(), network_signer: signer, - external_data_provider: Box::new(data_provider), db, }; @@ -1013,7 +922,7 @@ mod tests { init_logger(); let mut service1 = new_service(); - let service1_handle = service1.db_sync_handle(); + let service1_handle = service1.bitswap_handle(); // second service let service2 = NetworkServiceBuilder::new(); @@ -1027,17 +936,15 @@ mod tests { tokio::spawn(service1.loop_on_next()); tokio::spawn(service2.loop_on_next()); - let request = service1_handle.request(db_sync::Request::hashes([hello, world])); - let response = timeout(Duration::from_secs(5), request) - .await - .expect("time has elapsed") - .unwrap(); - assert_eq!( - response, - db_sync::Response::Hashes( - [(hello, b"hello".to_vec()), (world, b"world".to_vec())].into() - ) - ); + let hello_request = service1_handle.request(hello); + let world_request = service1_handle.request(world); + let response = timeout( + Duration::from_secs(5), + future::join(hello_request, world_request), + ) + .await + .expect("time has elapsed"); + assert_eq!(response, (b"hello".to_vec(), b"world".to_vec())); } #[tokio::test] @@ -1065,33 +972,6 @@ mod tests { assert_matches!(event, NetworkEvent::PeerBlocked(peer_id) if peer_id == service2_peer_id); } - #[tokio::test] - async fn external_data_provider() { - init_logger(); - - let alice = NetworkServiceBuilder::new(); - let alice_data_provider = alice.data_provider.clone(); - let mut alice = alice.build(); - let alice_handle = alice.db_sync_handle(); - - let bob = NetworkServiceBuilder::new(); - let bob_db = bob.db.clone(); - let mut bob = bob.build(); - - alice.connect(&mut bob).await; - tokio::spawn(alice.loop_on_next()); - tokio::spawn(bob.loop_on_next()); - - let expected_response = fill_data_provider(alice_data_provider, bob_db).await; - - let request = alice_handle.request(db_sync::Request::program_ids(H256::zero(), 2)); - let response = timeout(Duration::from_secs(5), request) - .await - .expect("time has elapsed") - .unwrap(); - assert_eq!(response, expected_response); - } - #[tokio::test] async fn validator_discovery() { init_logger(); diff --git a/ethexe/network/src/peer_score.rs b/ethexe/network/src/peer_score.rs index 87d41c56c0d..835723c55f6 100644 --- a/ethexe/network/src/peer_score.rs +++ b/ethexe/network/src/peer_score.rs @@ -45,14 +45,12 @@ struct Metrics { #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub(crate) enum ScoreDecreaseReason { - ExcessiveData, InvalidData, } impl ScoreDecreaseReason { fn to_i8(self, config: &Config) -> i8 { match self { - ScoreDecreaseReason::ExcessiveData => config.excessive_data, ScoreDecreaseReason::InvalidData => config.invalid_data, } } @@ -70,10 +68,6 @@ impl Handle { Self(tx) } - pub fn excessive_data(&self, peer_id: PeerId) { - let _res = self.0.send((peer_id, ScoreDecreaseReason::ExcessiveData)); - } - pub fn invalid_data(&self, peer_id: PeerId) { let _res = self.0.send((peer_id, ScoreDecreaseReason::InvalidData)); } @@ -98,7 +92,6 @@ pub(crate) enum Event { /// Behaviour config. #[derive(Debug, Clone)] pub(crate) struct Config { - excessive_data: i8, invalid_data: i8, decay: i8, blocked_threshold: i8, @@ -109,7 +102,6 @@ pub(crate) struct Config { impl Config { const fn new() -> Self { Self { - excessive_data: i8::MIN / 5, invalid_data: i8::MIN / 3, decay: i8::MAX / 17, blocked_threshold: i8::MIN / 3, @@ -379,12 +371,12 @@ mod tests { #[tokio::test(start_paused = true)] async fn smoke() { - const EXCESSIVE_DATA: i8 = Config::new().blocked_threshold / 3 - 1; + const INVALID_DATA: i8 = Config::new().blocked_threshold / 3 - 1; init_logger(); let alice_config = Config { - excessive_data: EXCESSIVE_DATA, + invalid_data: INVALID_DATA, ..Default::default() }; let mut alice = new_swarm_with_config(alice_config.clone()).await; @@ -394,37 +386,37 @@ mod tests { tokio::spawn(chad.loop_on_next()); let handle = alice.behaviour_mut().handle(); - handle.excessive_data(chad_peer_id); + handle.invalid_data(chad_peer_id); let event = future::poll_immediate(alice.next_behaviour_event()).await; assert_eq!(event, None); assert_eq!( alice.behaviour().get_score(chad_peer_id), - Some(EXCESSIVE_DATA) + Some(INVALID_DATA) ); - handle.excessive_data(chad_peer_id); + handle.invalid_data(chad_peer_id); let event = future::poll_immediate(alice.next_behaviour_event()).await; assert_eq!(event, None); assert_eq!( alice.behaviour().get_score(chad_peer_id), - Some(2 * EXCESSIVE_DATA) + Some(2 * INVALID_DATA) ); - handle.excessive_data(chad_peer_id); + handle.invalid_data(chad_peer_id); let event = alice.next_behaviour_event().await; assert_eq!( event, Event::PeerBlocked { peer_id: chad_peer_id, - last_reason: ScoreDecreaseReason::ExcessiveData + last_reason: ScoreDecreaseReason::InvalidData } ); assert_eq!( alice.behaviour().get_score(chad_peer_id), - Some(3 * EXCESSIVE_DATA) + Some(3 * INVALID_DATA) ); let event = alice.next_swarm_event().await; @@ -448,7 +440,7 @@ mod tests { ); assert_eq!( alice.behaviour().get_score(chad_peer_id), - Some(EXCESSIVE_DATA * 3 + alice_config.decay) + Some(INVALID_DATA * 3 + alice_config.decay) ); } diff --git a/ethexe/network/src/utils.rs b/ethexe/network/src/utils.rs index 5a9c0b7c7dd..1e4e2a4a7b2 100644 --- a/ethexe/network/src/utils.rs +++ b/ethexe/network/src/utils.rs @@ -16,17 +16,13 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::db_sync::PeerId; use async_trait::async_trait; use ip_network::IpNetwork; use libp2p::{ - Multiaddr, StreamProtocol, + Multiaddr, PeerId, StreamProtocol, futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, multiaddr, request_response, - swarm::{ - ConnectionClosed, ConnectionId, DialError, DialFailure, FromSwarm, NewExternalAddrOfPeer, - behaviour::ConnectionEstablished, - }, + swarm::{ConnectionId, DialError, DialFailure, FromSwarm, NewExternalAddrOfPeer}, }; use lru::LruCache; use parity_scale_codec::{Decode, DecodeAll, Encode}; @@ -215,29 +211,6 @@ impl ConnectionMap { limit: NoLimits, } } - - /// Returns true if a new connection added - pub(crate) fn on_swarm_event(&mut self, event: FromSwarm) -> bool { - match event { - FromSwarm::ConnectionEstablished(ConnectionEstablished { - peer_id, - connection_id, - .. - }) => { - let Ok(new) = self.add_connection(peer_id, connection_id); - new - } - FromSwarm::ConnectionClosed(ConnectionClosed { - peer_id, - connection_id, - .. - }) => { - self.remove_connection(peer_id, connection_id); - false - } - _ => false, - } - } } /// A helper struct for formatting collections (BTreeSet, BTreeMap) with two display modes: @@ -404,11 +377,8 @@ impl Default for PeerAddresses { #[cfg(test)] pub(crate) mod tests { - use crate::{ - db_sync::PeerId, - utils::{ConnectionMap, ExponentialBackoffInterval}, - }; - use libp2p::swarm::ConnectionId; + use crate::utils::{ConnectionMap, ExponentialBackoffInterval}; + use libp2p::{PeerId, swarm::ConnectionId}; use proptest::{ arbitrary::Arbitrary, strategy::{Strategy, ValueTree}, diff --git a/ethexe/network/src/validator/discovery.rs b/ethexe/network/src/validator/discovery.rs index ac8e4143312..14671e793c8 100644 --- a/ethexe/network/src/validator/discovery.rs +++ b/ethexe/network/src/validator/discovery.rs @@ -21,7 +21,6 @@ //! Heavily based on Substrate authority discovery mechanism. use crate::{ - db_sync::PeerId, kad::{ self, GetRecordResult, PutRecordFuture, RecordKey, ValidatorIdentityKey, ValidatorIdentityRecord, @@ -42,7 +41,7 @@ use futures::{ use gsigner::secp256k1::{PrivateKey, Secp256k1SignerExt, Signer}; use indexmap::IndexSet; use libp2p::{ - Multiaddr, + Multiaddr, PeerId, core::{Endpoint, transport::PortUse}, identity::Keypair, multiaddr, diff --git a/ethexe/network/src/validator/topic.rs b/ethexe/network/src/validator/topic.rs index 52e3b8a302a..4dbfc465c0d 100644 --- a/ethexe/network/src/validator/topic.rs +++ b/ethexe/network/src/validator/topic.rs @@ -19,15 +19,13 @@ //! Validator-specific networking logic that verifies signed messages //! against on-chain state. -use crate::{ - db_sync::PeerId, gossipsub::MessageAcceptance, peer_score, - validator::list::ValidatorListSnapshot, -}; +use crate::{gossipsub::MessageAcceptance, peer_score, validator::list::ValidatorListSnapshot}; use ethexe_common::{ Address, HashOf, injected::{InjectedTransaction, SignedCompactPromise}, network::VerifiedValidatorMessage, }; +use libp2p::PeerId; use lru::LruCache; use std::{cmp::Ordering, collections::VecDeque, mem, num::NonZeroUsize, sync::Arc}; diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 38d8b0409ff..2fc926e1844 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -44,7 +44,7 @@ use ethexe_db::{ visitor::DatabaseVisitor, }; use ethexe_ethereum::mirror::MirrorQuery; -use ethexe_network::{NetworkService, db_sync}; +use ethexe_network::NetworkService; use ethexe_observer::{ ObserverService, utils::{BlockId, BlockLoader}, diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 459809b9ba1..7042f388efc 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -53,10 +53,9 @@ use alloy::{ rpc::types::anvil::Metadata, }; use anyhow::{Context, Result, bail}; -use async_trait::async_trait; use ethexe_blob_loader::{BlobLoader, BlobLoaderEvent, BlobLoaderService, ConsensusLayerConfig}; use ethexe_common::{ - COMMITMENT_DELAY_LIMIT, CodeAndIdUnchecked, PromiseEmissionMode, gear::CodeState, + COMMITMENT_DELAY_LIMIT, CodeAndIdUnchecked, PromiseEmissionMode, network::VerifiedValidatorMessage, }; use ethexe_compute::{ComputeConfig, ComputeEvent, ComputeService}; @@ -67,10 +66,7 @@ use ethexe_db::{ Database, GenesisInitializer, InitConfig, RawDatabase, RocksDatabase, dump::StateDump, }; use ethexe_ethereum::{EthereumBuilder, deploy::EthereumDeployer, router::RouterQuery}; -use ethexe_network::{ - NetworkEvent, NetworkRuntimeConfig, NetworkService, - db_sync::{self, ExternalDataProvider}, -}; +use ethexe_network::{NetworkEvent, NetworkRuntimeConfig, NetworkService}; use ethexe_observer::{ ObserverConfig, ObserverEvent, ObserverService, utils::{BlockId, BlockLoader}, @@ -80,15 +76,9 @@ use ethexe_prometheus::{PrometheusEvent, PrometheusService}; use ethexe_rpc::{RpcEvent, RpcServer}; use ethexe_service_utils::{OptionFuture as _, OptionStreamNext as _}; use futures::{FutureExt, StreamExt, stream::FuturesUnordered}; -use gprimitives::{ActorId, CodeId, H256}; +use gprimitives::CodeId; use gsigner::secp256k1::{Address, PrivateKey, PublicKey, Signer}; -use std::{ - collections::{BTreeSet, HashMap}, - num::NonZero, - path::PathBuf, - pin::Pin, - time::Duration, -}; +use std::{collections::HashMap, num::NonZero, path::PathBuf, pin::Pin, time::Duration}; use tokio::sync::oneshot; pub mod config; @@ -109,32 +99,6 @@ pub enum Event { Fetching(db_sync::HandleResult), } -#[derive(Clone)] -struct RouterDataProvider(RouterQuery); - -#[async_trait] -impl ExternalDataProvider for RouterDataProvider { - fn clone_boxed(&self) -> Box { - Box::new(self.clone()) - } - - async fn programs_code_ids_at( - self: Box, - program_ids: BTreeSet, - block: H256, - ) -> Result> { - self.0.programs_code_ids_at(program_ids, block).await - } - - async fn codes_states_at( - self: Box, - code_ids: BTreeSet, - block: H256, - ) -> Result> { - self.0.codes_states_at(code_ids, block).await - } -} - /// ethexe service. pub struct Service { db: Database, @@ -450,7 +414,6 @@ impl Service { validator_key: validator_pub_key, general_signer: signer.clone(), network_signer, - external_data_provider: Box::new(RouterDataProvider(router_query)), db: db.clone(), }; @@ -803,20 +766,7 @@ impl Service { unreachable!("Fetching event is impossible without network service"); }; - match result { - Ok(db_sync::Response::Announces(response)) => { - consensus.receive_announces_response(response)?; - } - Ok(resp) => { - panic!("only announces are requested currently, but got: {resp:?}"); - } - Err((err, request)) => { - log::trace!( - "Retry fetching external data for request {request:?} due to error: {err:?}" - ); - network_fetcher.push(network.db_sync_handle().retry(request)); - } - } + consensus.receive_announces_response(result)?; } } } From 7b26975a32ef29a7fd8a53274cba6645050b815b Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 13 May 2026 19:58:35 +0300 Subject: [PATCH 02/72] Initial announces --- ethexe/common/src/network.rs | 131 +++++++++++++++++++++++++++++++- ethexe/network/src/lib.rs | 13 ++-- ethexe/service/src/fast_sync.rs | 93 ++++++++++------------- ethexe/service/src/lib.rs | 23 ++++-- 4 files changed, 193 insertions(+), 67 deletions(-) diff --git a/ethexe/common/src/network.rs b/ethexe/common/src/network.rs index de608fb5e32..9900a1f11fb 100644 --- a/ethexe/common/src/network.rs +++ b/ethexe/common/src/network.rs @@ -21,8 +21,9 @@ use crate::{ consensus::{BatchCommitmentValidationReply, BatchCommitmentValidationRequest}, ecdsa::{SignedData, VerifiedData}, }; -use alloc::vec::Vec; +use alloc::{collections::VecDeque, vec::Vec}; use core::{hash::Hash, num::NonZeroU32}; +use gprimitives::H256; use parity_scale_codec::{Decode, Encode}; use sha3::Keccak256; @@ -89,6 +90,11 @@ impl VerifiedValidatorMessage { } } +#[allow(async_fn_in_trait)] +pub trait BitswapHandle { + async fn request(&self, hash: H256) -> Vec; +} + /// Until condition for announces request (see [`AnnouncesRequest`]). #[derive(PartialEq, Eq, Hash, Debug, Clone, Copy, Encode, Decode, derive_more::From)] pub enum AnnouncesRequestUntil { @@ -113,15 +119,16 @@ pub struct AnnouncesRequest { #[derive(derive_more::Debug, Clone, Eq, PartialEq, derive_more::From)] pub struct AnnouncesResponse { /// Corresponding request for this response - request: AnnouncesRequest, + pub request: AnnouncesRequest, /// List of announces - announces: Vec, + pub announces: Vec, } impl AnnouncesResponse { /// # Safety /// - /// Response must be only created by network service + /// Response must be only created after checking that the announce chain + /// matches the corresponding request. pub unsafe fn from_parts(request: AnnouncesRequest, announces: Vec) -> Self { Self { request, announces } } @@ -138,3 +145,119 @@ impl AnnouncesResponse { (self.request, self.announces) } } + +#[derive(Debug, Clone, PartialEq, Eq, derive_more::Display)] +pub enum AnnouncesRequestError { + #[display("requested chain length {_0} exceeds maximum allowed {_1}")] + ChainLenExceedsMax(NonZeroU32, NonZeroU32), + #[display("announce {_0} failed to decode")] + DecodeFailed(HashOf), + #[display("announces response is empty")] + EmptyResponse, + #[display("announces response head mismatch: expected {expected}, received {received}")] + HeadMismatch { + expected: HashOf, + received: HashOf, + }, + #[display("announces response tail mismatch: expected {expected}, received {received}")] + TailMismatch { + expected: HashOf, + received: HashOf, + }, + #[display("announces response length mismatch: expected {expected}, received {received}")] + LenMismatch { expected: usize, received: usize }, + #[display("announces response chain is not linked")] + ChainIsNotLinked, + #[display("reached maximum chain length {_0}")] + ReachedMaxChainLen(NonZeroU32), +} + +#[cfg(feature = "std")] +impl std::error::Error for AnnouncesRequestError {} + +pub async fn request_announces( + bitswap: &impl BitswapHandle, + request: AnnouncesRequest, + max_chain_len: NonZeroU32, +) -> Result { + if let AnnouncesRequestUntil::ChainLen(len) = request.until + && len > max_chain_len + { + return Err(AnnouncesRequestError::ChainLenExceedsMax( + len, + max_chain_len, + )); + } + + let mut announces = VecDeque::new(); + let mut announce_hash = request.head; + + loop { + match request.until { + AnnouncesRequestUntil::Tail(tail) if announce_hash == tail => { + return validate_announces(request, announces.into()); + } + AnnouncesRequestUntil::ChainLen(len) if announces.len() == len.get() as usize => { + return validate_announces(request, announces.into()); + } + _ => {} + } + + if announces.len() == max_chain_len.get() as usize { + return Err(AnnouncesRequestError::ReachedMaxChainLen(max_chain_len)); + } + + let data = bitswap.request(announce_hash.inner()).await; + let announce = Announce::decode(&mut data.as_slice()) + .map_err(|_| AnnouncesRequestError::DecodeFailed(announce_hash))?; + + announce_hash = announce.parent; + announces.push_front(announce); + } +} + +fn validate_announces( + request: AnnouncesRequest, + announces: Vec, +) -> Result { + let Some((first, last)) = announces.first().zip(announces.last()) else { + return Err(AnnouncesRequestError::EmptyResponse); + }; + + if request.head != last.to_hash() { + return Err(AnnouncesRequestError::HeadMismatch { + expected: request.head, + received: last.to_hash(), + }); + } + + match request.until { + AnnouncesRequestUntil::Tail(request_tail_hash) => { + if request_tail_hash != first.parent { + return Err(AnnouncesRequestError::TailMismatch { + expected: request_tail_hash, + received: first.parent, + }); + } + } + AnnouncesRequestUntil::ChainLen(len) => { + if announces.len() != len.get() as usize { + return Err(AnnouncesRequestError::LenMismatch { + expected: len.get() as usize, + received: announces.len(), + }); + } + } + } + + // Check chain linking + let mut expected_parent_hash = first.parent; + for announce in announces.iter() { + if announce.parent != expected_parent_hash { + return Err(AnnouncesRequestError::ChainIsNotLinked); + } + expected_parent_hash = announce.to_hash(); + } + + Ok(unsafe { AnnouncesResponse::from_parts(request, announces) }) +} diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index a39146f2c93..d0f804a1701 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -59,7 +59,7 @@ use ethexe_common::{ db::ConfigStorageRO, ecdsa::PublicKey, injected::{AddressedInjectedTransaction, SignedCompactPromise}, - network::{SignedValidatorMessage, VerifiedValidatorMessage}, + network::{BitswapHandle, SignedValidatorMessage, VerifiedValidatorMessage}, }; use ethexe_db::Database; use futures::{Stream, future::Either, ready, stream::FusedStream}; @@ -103,6 +103,12 @@ const MAX_PENDING_OUTGOING_CONNECTIONS: u32 = 10; /// response. pub const DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE: NonZeroU32 = NonZeroU32::new(1000).unwrap(); +impl BitswapHandle for bitswap::Handle { + async fn request(&self, hash: H256) -> Vec { + bitswap::Handle::request(self, hash).await + } +} + /// High-level events produced by [`NetworkService`]. #[derive(derive_more::Debug)] pub enum NetworkEvent { @@ -264,7 +270,7 @@ impl NetworkService { transport_type, router_address, allow_non_global_addresses, - max_chain_len_for_announces_response, + max_chain_len_for_announces_response: _, } = config; let NetworkRuntimeConfig { @@ -301,7 +307,6 @@ impl NetworkService { general_signer, validator_list_snapshot: validator_list_snapshot.clone(), allow_non_global_addresses, - max_chain_len_for_announces_response, metrics: (&mut registry, metrics.clone()), }; let behaviour = Behaviour::new(behaviour_config)?; @@ -703,7 +708,6 @@ struct BehaviourConfig<'a> { general_signer: Signer, validator_list_snapshot: Arc, allow_non_global_addresses: bool, - max_chain_len_for_announces_response: NonZeroU32, metrics: ( &'a mut libp2p::metrics::Registry, Arc, @@ -747,7 +751,6 @@ impl Behaviour { general_signer, validator_list_snapshot, allow_non_global_addresses, - max_chain_len_for_announces_response, metrics: (registry, metrics), } = config; diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 2fc926e1844..ac1a18facf6 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -31,7 +31,7 @@ use ethexe_common::{ router::{AnnouncesCommittedEvent, BatchCommittedEvent}, }, injected, - network::{AnnouncesRequest, AnnouncesRequestUntil}, + network::{AnnouncesRequest, AnnouncesRequestUntil, request_announces}, }; use ethexe_compute::ComputeService; use ethexe_db::{ @@ -44,7 +44,7 @@ use ethexe_db::{ visitor::DatabaseVisitor, }; use ethexe_ethereum::mirror::MirrorQuery; -use ethexe_network::NetworkService; +use ethexe_network::{DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, NetworkService}; use ethexe_observer::{ ObserverService, utils::{BlockId, BlockLoader}, @@ -129,24 +129,34 @@ impl EventData { } } -async fn net_fetch( +async fn bitswap_fetch(network: &mut NetworkService, hash: H256) -> Vec { + let bitswap = network.bitswap_handle(); + let fut = bitswap.request(hash); + tokio::pin!(fut); + loop { + tokio::select! { + _ = network.select_next_some() => {}, + data = &mut fut => break data, + } + } +} + +async fn bitswap_fetch_announces( network: &mut NetworkService, - request: db_sync::Request, -) -> Result { - let mut fut = network.db_sync_handle().request(request); + request: AnnouncesRequest, +) -> Result { + let bitswap = network.bitswap_handle(); + let fut = request_announces( + &bitswap, + request, + DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, + ); + tokio::pin!(fut); + loop { tokio::select! { _ = network.select_next_some() => {}, - res = &mut fut => { - match res { - Ok(response) => break Ok(response), - Err((err, request)) => { - log::warn!("Request {:?} failed: {err}. Retrying...", request.id()); - fut = network.db_sync_handle().retry(request); - continue; - } - } - } + response = &mut fut => break response.map_err(Into::into), } } } @@ -154,22 +164,15 @@ async fn net_fetch( /// Collects program code IDs for the latest committed block. async fn collect_program_code_ids( observer: &mut ObserverService, - network: &mut NetworkService, + _network: &mut NetworkService, latest_committed_block: H256, ) -> Result> { let router_query = observer.router_query(); - let programs_count = router_query + let _programs_count = router_query .programs_count_at(latest_committed_block) .await?; - let response = net_fetch( - network, - db_sync::Request::program_ids(latest_committed_block, programs_count), - ) - .await?; - - let program_code_ids = response.unwrap_program_ids(); - Ok(program_code_ids) + anyhow::bail!("fast-sync program id enumeration via bitswap is not implemented yet") } async fn collect_announce( @@ -181,16 +184,14 @@ async fn collect_announce( return Ok(announce); } - let response = net_fetch( + let response = bitswap_fetch_announces( network, AnnouncesRequest { head: announce_hash, until: AnnouncesRequestUntil::ChainLen(NonZeroU32::MIN), - } - .into(), + }, ) - .await? - .unwrap_announces(); + .await?; // Response is checked so we can just take the first announce let (_, mut announces) = response.into_parts(); @@ -200,28 +201,17 @@ async fn collect_announce( /// Collects a set of valid code IDs that are not yet validated in the local database. async fn collect_code_ids( observer: &mut ObserverService, - network: &mut NetworkService, + _network: &mut NetworkService, db: &Database, latest_committed_block: H256, ) -> Result> { let router_query = observer.router_query(); - let codes_count = router_query + let _codes_count = router_query .validated_codes_count_at(latest_committed_block) .await?; - let response = net_fetch( - network, - db_sync::Request::valid_codes(latest_committed_block, codes_count), - ) - .await?; - - let code_ids = response - .unwrap_valid_codes() - .into_iter() - .filter(|&code_id| db.code_valid(code_id).is_none()) - .collect(); - - Ok(code_ids) + let _ = db; + anyhow::bail!("fast-sync valid code enumeration via bitswap is not implemented yet") } /// Collects the program states for a given set of program IDs at a specified block height. @@ -324,10 +314,10 @@ impl RequestManager { let pending_network_requests = self.handle_pending_requests(); if !pending_network_requests.is_empty() { - let request: BTreeSet = pending_network_requests.keys().copied().collect(); - let response = net_fetch(network, db_sync::Request::hashes(request)) - .await - .expect("no external validation required"); + let mut response = Vec::with_capacity(pending_network_requests.len()); + for &hash in pending_network_requests.keys() { + response.push((hash, bitswap_fetch(network, hash).await)); + } self.handle_response(pending_network_requests, response); } @@ -364,9 +354,8 @@ impl RequestManager { fn handle_response( &mut self, mut pending_network_requests: HashMap, - response: db_sync::Response, + data: Vec<(H256, Vec)>, ) { - let data = response.unwrap_hashes(); for (hash, data) in data { let metadata = pending_network_requests .remove(&hash) diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 7042f388efc..954a0e43b77 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -56,7 +56,7 @@ use anyhow::{Context, Result, bail}; use ethexe_blob_loader::{BlobLoader, BlobLoaderEvent, BlobLoaderService, ConsensusLayerConfig}; use ethexe_common::{ COMMITMENT_DELAY_LIMIT, CodeAndIdUnchecked, PromiseEmissionMode, - network::VerifiedValidatorMessage, + network::{AnnouncesRequestError, VerifiedValidatorMessage, request_announces}, }; use ethexe_compute::{ComputeConfig, ComputeEvent, ComputeService}; use ethexe_consensus::{ @@ -66,7 +66,10 @@ use ethexe_db::{ Database, GenesisInitializer, InitConfig, RawDatabase, RocksDatabase, dump::StateDump, }; use ethexe_ethereum::{EthereumBuilder, deploy::EthereumDeployer, router::RouterQuery}; -use ethexe_network::{NetworkEvent, NetworkRuntimeConfig, NetworkService}; +use ethexe_network::{ + DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, NetworkEvent, NetworkRuntimeConfig, + NetworkService, +}; use ethexe_observer::{ ObserverConfig, ObserverEvent, ObserverService, utils::{BlockId, BlockLoader}, @@ -96,7 +99,7 @@ pub enum Event { BlobLoader(BlobLoaderEvent), Rpc(RpcEvent), Prometheus(PrometheusEvent), - Fetching(db_sync::HandleResult), + Fetching(Result), } /// ethexe service. @@ -743,7 +746,15 @@ impl Service { panic!("Requesting announces is not allowed without network service"); }; - network_fetcher.push(network.db_sync_handle().request(request.into())); + let bitswap = network.bitswap_handle(); + network_fetcher.push(async move { + request_announces( + &bitswap, + request, + DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, + ) + .await + }); } ConsensusEvent::AnnounceAccepted(_) | ConsensusEvent::AnnounceRejected(_) => { // TODO #4940: consider to publish network message @@ -762,11 +773,11 @@ impl Service { } }, Event::Fetching(result) => { - let Some(network) = network.as_mut() else { + if network.is_none() { unreachable!("Fetching event is impossible without network service"); }; - consensus.receive_announces_response(result)?; + consensus.receive_announces_response(result?)?; } } } From cf5582d701bf0a38cb0e3fe5c2a69e3708f68ca9 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 13 May 2026 20:58:50 +0300 Subject: [PATCH 03/72] Done announces --- ethexe/cli/src/params/network.rs | 5 +- ethexe/common/src/network.rs | 172 ++++------------------ ethexe/consensus/src/connect/mod.rs | 2 +- ethexe/consensus/src/validator/initial.rs | 69 ++++----- ethexe/network/src/bitswap.rs | 156 ++++++++++++++------ ethexe/network/src/lib.rs | 32 ++-- ethexe/service/src/fast_sync.rs | 19 +-- ethexe/service/src/lib.rs | 20 +-- ethexe/service/src/tests/utils/env.rs | 4 +- 9 files changed, 199 insertions(+), 280 deletions(-) diff --git a/ethexe/cli/src/params/network.rs b/ethexe/cli/src/params/network.rs index 294acaf24d1..20294244e06 100644 --- a/ethexe/cli/src/params/network.rs +++ b/ethexe/cli/src/params/network.rs @@ -23,7 +23,7 @@ use anyhow::{Context, Result}; use clap::Parser; use ethexe_common::Address; use ethexe_network::{ - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, NetworkConfig, + NetworkConfig, export::{Multiaddr, Protocol}, }; use gsigner::secp256k1::Signer; @@ -146,9 +146,6 @@ impl NetworkParams { listen_addresses, transport_type: Default::default(), allow_non_global_addresses: is_dev, - max_chain_len_for_announces_response: self - .max_chain_len_for_announces_response - .unwrap_or(DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE), })) } } diff --git a/ethexe/common/src/network.rs b/ethexe/common/src/network.rs index 9900a1f11fb..c8c29046d6a 100644 --- a/ethexe/common/src/network.rs +++ b/ethexe/common/src/network.rs @@ -23,7 +23,6 @@ use crate::{ }; use alloc::{collections::VecDeque, vec::Vec}; use core::{hash::Hash, num::NonZeroU32}; -use gprimitives::H256; use parity_scale_codec::{Decode, Encode}; use sha3::Keccak256; @@ -91,8 +90,8 @@ impl VerifiedValidatorMessage { } #[allow(async_fn_in_trait)] -pub trait BitswapHandle { - async fn request(&self, hash: H256) -> Vec; +pub trait BitswapAnnouncesHandle { + async fn request(&self, hash: HashOf) -> Announce; } /// Until condition for announces request (see [`AnnouncesRequest`]). @@ -115,6 +114,35 @@ pub struct AnnouncesRequest { pub until: AnnouncesRequestUntil, } +impl AnnouncesRequest { + pub async fn request(self, bitswap: impl BitswapAnnouncesHandle) -> AnnouncesResponse { + let mut announces = VecDeque::new(); + let mut announce_hash = self.head; + + loop { + match self.until { + AnnouncesRequestUntil::Tail(tail) if announce_hash == tail => { + return AnnouncesResponse { + request: self, + announces: announces.into(), + }; + } + AnnouncesRequestUntil::ChainLen(len) if announces.len() == len.get() as usize => { + return AnnouncesResponse { + request: self, + announces: announces.into(), + }; + } + _ => {} + } + + let announce = bitswap.request(announce_hash).await; + announce_hash = announce.parent; + announces.push_front(announce); + } + } +} + /// Checked announces response ensuring that it matches the corresponding request. #[derive(derive_more::Debug, Clone, Eq, PartialEq, derive_more::From)] pub struct AnnouncesResponse { @@ -123,141 +151,3 @@ pub struct AnnouncesResponse { /// List of announces pub announces: Vec, } - -impl AnnouncesResponse { - /// # Safety - /// - /// Response must be only created after checking that the announce chain - /// matches the corresponding request. - pub unsafe fn from_parts(request: AnnouncesRequest, announces: Vec) -> Self { - Self { request, announces } - } - - pub fn request(&self) -> &AnnouncesRequest { - &self.request - } - - pub fn announces(&self) -> &[Announce] { - &self.announces - } - - pub fn into_parts(self) -> (AnnouncesRequest, Vec) { - (self.request, self.announces) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, derive_more::Display)] -pub enum AnnouncesRequestError { - #[display("requested chain length {_0} exceeds maximum allowed {_1}")] - ChainLenExceedsMax(NonZeroU32, NonZeroU32), - #[display("announce {_0} failed to decode")] - DecodeFailed(HashOf), - #[display("announces response is empty")] - EmptyResponse, - #[display("announces response head mismatch: expected {expected}, received {received}")] - HeadMismatch { - expected: HashOf, - received: HashOf, - }, - #[display("announces response tail mismatch: expected {expected}, received {received}")] - TailMismatch { - expected: HashOf, - received: HashOf, - }, - #[display("announces response length mismatch: expected {expected}, received {received}")] - LenMismatch { expected: usize, received: usize }, - #[display("announces response chain is not linked")] - ChainIsNotLinked, - #[display("reached maximum chain length {_0}")] - ReachedMaxChainLen(NonZeroU32), -} - -#[cfg(feature = "std")] -impl std::error::Error for AnnouncesRequestError {} - -pub async fn request_announces( - bitswap: &impl BitswapHandle, - request: AnnouncesRequest, - max_chain_len: NonZeroU32, -) -> Result { - if let AnnouncesRequestUntil::ChainLen(len) = request.until - && len > max_chain_len - { - return Err(AnnouncesRequestError::ChainLenExceedsMax( - len, - max_chain_len, - )); - } - - let mut announces = VecDeque::new(); - let mut announce_hash = request.head; - - loop { - match request.until { - AnnouncesRequestUntil::Tail(tail) if announce_hash == tail => { - return validate_announces(request, announces.into()); - } - AnnouncesRequestUntil::ChainLen(len) if announces.len() == len.get() as usize => { - return validate_announces(request, announces.into()); - } - _ => {} - } - - if announces.len() == max_chain_len.get() as usize { - return Err(AnnouncesRequestError::ReachedMaxChainLen(max_chain_len)); - } - - let data = bitswap.request(announce_hash.inner()).await; - let announce = Announce::decode(&mut data.as_slice()) - .map_err(|_| AnnouncesRequestError::DecodeFailed(announce_hash))?; - - announce_hash = announce.parent; - announces.push_front(announce); - } -} - -fn validate_announces( - request: AnnouncesRequest, - announces: Vec, -) -> Result { - let Some((first, last)) = announces.first().zip(announces.last()) else { - return Err(AnnouncesRequestError::EmptyResponse); - }; - - if request.head != last.to_hash() { - return Err(AnnouncesRequestError::HeadMismatch { - expected: request.head, - received: last.to_hash(), - }); - } - - match request.until { - AnnouncesRequestUntil::Tail(request_tail_hash) => { - if request_tail_hash != first.parent { - return Err(AnnouncesRequestError::TailMismatch { - expected: request_tail_hash, - received: first.parent, - }); - } - } - AnnouncesRequestUntil::ChainLen(len) => { - if announces.len() != len.get() as usize { - return Err(AnnouncesRequestError::LenMismatch { - expected: len.get() as usize, - received: announces.len(), - }); - } - } - } - - // Check chain linking - let mut expected_parent_hash = first.parent; - for announce in announces.iter() { - if announce.parent != expected_parent_hash { - return Err(AnnouncesRequestError::ChainIsNotLinked); - } - expected_parent_hash = announce.to_hash(); - } - - Ok(unsafe { AnnouncesResponse::from_parts(request, announces) }) -} diff --git a/ethexe/consensus/src/connect/mod.rs b/ethexe/consensus/src/connect/mod.rs index 80b6d7aaf63..1e1a85460ae 100644 --- a/ethexe/consensus/src/connect/mod.rs +++ b/ethexe/consensus/src/connect/mod.rs @@ -328,7 +328,7 @@ impl ConsensusService for ConnectService { let block = *block; let producer = *producer; - let (request, announces) = response.into_parts(); + let AnnouncesResponse { request, announces } = response; if waiting_request != &request { return Ok(()); diff --git a/ethexe/consensus/src/validator/initial.rs b/ethexe/consensus/src/validator/initial.rs index 13d56f8b83c..548a464243a 100644 --- a/ethexe/consensus/src/validator/initial.rs +++ b/ethexe/consensus/src/validator/initial.rs @@ -174,12 +174,11 @@ impl StateHandler for Initial { block, chain, announces, - } if announces == *response.request() => { + } if announces == response.request => { tracing::debug!(block = %block.hash, "Received missing announces response"); let missing_announces = response - .into_parts() - .1 + .announces .into_iter() .map(|a| (a.to_hash(), a)) .collect(); @@ -382,20 +381,18 @@ mod tests { }; assert_eq!(state.context().output, vec![expected_request.into()]); - let response = unsafe { - AnnouncesResponse::from_parts( - expected_request, - vec![ - chain - .announces - .get(&chain.block_top_announce_hash(last - 3)) - .unwrap() - .announce - .clone(), - announce2.clone(), - announce1.clone(), - ], - ) + let response = AnnouncesResponse { + request: expected_request, + announces: vec![ + chain + .announces + .get(&chain.block_top_announce_hash(last - 3)) + .unwrap() + .announce + .clone(), + announce2.clone(), + announce1.clone(), + ], }; // In successful case no new events are produced @@ -566,14 +563,12 @@ mod tests { let invalid_announce = Announce::base(H256::random(), HashOf::random()); let invalid_announce_hash = invalid_announce.to_hash(); - let response = unsafe { - AnnouncesResponse::from_parts( - AnnouncesRequest { - head: invalid_announce_hash, - until: NonZeroU32::new(1).unwrap().into(), - }, - vec![invalid_announce], - ) + let response = AnnouncesResponse { + request: AnnouncesRequest { + head: invalid_announce_hash, + until: NonZeroU32::new(1).unwrap().into(), + }, + announces: vec![invalid_announce], }; let state = Initial::create_with_chain_head(ctx, block) @@ -642,19 +637,17 @@ mod tests { }; assert_eq!(state.context().output, vec![expected_request.into()]); - let response = unsafe { - AnnouncesResponse::from_parts( - expected_request, - vec![ - chain - .announces - .get(&chain.block_top_announce_hash(last - 7)) - .unwrap() - .announce - .clone(), - unknown_announce, - ], - ) + let response = AnnouncesResponse { + request: expected_request, + announces: vec![ + chain + .announces + .get(&chain.block_top_announce_hash(last - 7)) + .unwrap() + .announce + .clone(), + unknown_announce, + ], }; let state = state.process_announces_response(response).unwrap(); diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs index f96302d3436..cb4057252f5 100644 --- a/ethexe/network/src/bitswap.rs +++ b/ethexe/network/src/bitswap.rs @@ -19,7 +19,10 @@ use beetswap::multihasher::{Multihasher, MultihasherError}; use blockstore::{block::CidError, cond_send::CondSend}; use cid::{Cid, CidGeneric}; -use ethexe_common::db::HashStorageRO; +use ethexe_common::{ + Announce, HashOf, + db::{AnnounceStorageRO, HashStorageRO}, +}; use futures::FutureExt; use gprimitives::H256; use libp2p::{ @@ -31,6 +34,7 @@ use libp2p::{ }, }; use multihash::Multihash; +use parity_scale_codec::{Decode, Encode}; use std::{ collections::HashMap, mem, @@ -42,22 +46,62 @@ use tokio::{ task, }; +#[derive(Debug, Copy, Clone, Eq, PartialEq, derive_more::From)] +pub enum Request { + Hash(H256), + Announce(HashOf), +} + +impl Request { + fn into_cid(self) -> Cid { + match self { + Request::Hash(hash) => Cid::new_v1( + Blockstore::RAW_CODEC, + Multihash::wrap(Blockstore::BLAKE2B_CODE, hash.as_bytes()) + .expect("size is always correct"), + ), + Request::Announce(hash) => Cid::new_v1( + Blockstore::ANNOUNCES_CODEC, + Multihash::wrap(Blockstore::BLAKE2B_CODE, hash.inner().as_bytes()) + .expect("size is always correct"), + ), + } + } + + fn into_response(self, data: Vec) -> Response { + match self { + Request::Hash(_) => Response::Hash(data), + Request::Announce(_) => { + Response::Announce(Announce::decode(&mut data.as_slice()).expect("valid announce")) + } + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, derive_more::Unwrap)] +pub enum Response { + Hash(Vec), + Announce(Announce), +} + #[derive(Clone)] -pub struct Handle(mpsc::UnboundedSender<(H256, oneshot::Sender>)>); +pub struct Handle(mpsc::UnboundedSender<(Request, oneshot::Sender)>); impl Handle { - pub async fn request(&self, request: H256) -> Vec { + pub async fn request(&self, request: impl Into) -> Response { let (tx, rx) = oneshot::channel(); self.0 - .send((request, tx)) + .send((request.into(), tx)) .expect("channel should never be closed"); rx.await.expect("channel should never be closed") } } -pub(crate) trait BlockstoreDatabase: Send + Sync + HashStorageRO { +pub(crate) trait BlockstoreDatabase: + Send + Sync + HashStorageRO + AnnounceStorageRO +{ fn clone_boxed(&self) -> Box; } @@ -74,7 +118,25 @@ pub struct Blockstore { impl Blockstore { const MAX_BLOCK_SIZE: u64 = 1024 * 1024; // 1MB const BLAKE2B_CODE: u64 = 0xb220; - const CID_CODEC: u64 = 0x55; + const RAW_CODEC: u64 = 0x55; + const ANNOUNCES_CODEC: u64 = 0x300000; + + fn convert_multihash(multihash: &Multihash) -> blockstore::Result { + let hash: Multihash<32> = + beetswap::utils::convert_multihash(multihash).ok_or(blockstore::Error::CidTooLarge)?; + if hash.code() != Self::BLAKE2B_CODE { + return Err(blockstore::Error::CidError(CidError::InvalidMultihashCode( + hash.code(), + Self::BLAKE2B_CODE, + ))); + } + if hash.size() as usize != mem::size_of::() { + return Err(blockstore::Error::CidError( + CidError::InvalidMultihashLength(hash.size() as usize), + )); + } + Ok(H256::from_slice(hash.digest())) + } } impl blockstore::Blockstore for Blockstore { @@ -82,34 +144,41 @@ impl blockstore::Blockstore for Blockstore { &self, cid: &CidGeneric, ) -> impl Future>>> + CondSend { - let hash = *cid.hash(); let db = self.db.clone_boxed(); + let hash = *cid.hash(); + let codec = cid.codec(); task::spawn_blocking(move || { - let hash: Multihash<32> = - beetswap::utils::convert_multihash(&hash).ok_or(blockstore::Error::CidTooLarge)?; - if hash.code() != Self::BLAKE2B_CODE { - return Err(blockstore::Error::CidError(CidError::InvalidMultihashCode( - hash.code(), - Self::BLAKE2B_CODE, - ))); - } - if hash.size() as usize != mem::size_of::() { - return Err(blockstore::Error::CidError( - CidError::InvalidMultihashLength(hash.size() as usize), - )); - } - - let hash = H256::from_slice(hash.digest()); - let data = db.read_by_hash(hash); - - if let Some(data) = &data - && data.len() as u64 > Self::MAX_BLOCK_SIZE - { - log::warn!("{hash} is too large: {} bytes", data.len()); - return Err(blockstore::Error::ValueTooLarge); + let hash = Self::convert_multihash(&hash)?; + match codec { + Self::RAW_CODEC => { + let data = db.read_by_hash(hash); + + if let Some(data) = &data + && data.len() as u64 > Self::MAX_BLOCK_SIZE + { + log::warn!("{hash} is too large: {} bytes", data.len()); + return Err(blockstore::Error::ValueTooLarge); + } + + Ok(data) + } + Self::ANNOUNCES_CODEC => { + let hash = unsafe { HashOf::new(hash) }; + let announce = db.announce(hash); + + if let Some(announce) = &announce + && announce.encoded_size() as u64 > Self::MAX_BLOCK_SIZE + { + log::warn!("{hash} is too large: {} bytes", announce.encoded_size()); + return Err(blockstore::Error::ValueTooLarge); + } + + Ok(announce.map(|announce| announce.encode())) + } + codec => Err(blockstore::Error::CidError(CidError::InvalidCidCodec( + codec, + ))), } - - Ok(data) }) .map(|res| res.expect("database panicked")) } @@ -155,8 +224,8 @@ type InnerBehaviour = beetswap::Behaviour<32, Blockstore>; pub struct Behaviour { inner: InnerBehaviour, handle: Handle, - rx: mpsc::UnboundedReceiver<(H256, oneshot::Sender>)>, - requests: HashMap>>, + rx: mpsc::UnboundedReceiver<(Request, oneshot::Sender)>, + requests: HashMap)>, } impl Behaviour { @@ -180,19 +249,12 @@ impl Behaviour { self.handle.clone() } - fn cid(hash: H256) -> Cid { - Cid::new_v1( - Blockstore::CID_CODEC, - Multihash::wrap(Blockstore::BLAKE2B_CODE, hash.as_bytes()) - .expect("size is always correct"), - ) - } - fn handle_inner_event(&mut self, event: beetswap::Event) { match event { beetswap::Event::GetQueryResponse { query_id, data } => { - if let Some(channel) = self.requests.remove(&query_id) { - let _ = channel.send(data); + if let Some((request, channel)) = self.requests.remove(&query_id) { + let response = request.into_response(data); + let _ = channel.send(response); } } beetswap::Event::GetQueryError { query_id, error } => { @@ -283,7 +345,7 @@ impl NetworkBehaviour for Behaviour { &mut self, cx: &mut Context<'_>, ) -> Poll>> { - self.requests.retain(|&query_id, channel| { + self.requests.retain(|&query_id, (_, channel)| { if channel.is_closed() { self.inner.cancel(query_id); return false; @@ -292,10 +354,10 @@ impl NetworkBehaviour for Behaviour { true }); - while let Poll::Ready(Some((hash, channel))) = self.rx.poll_recv(cx) { - let cid = Self::cid(hash); + while let Poll::Ready(Some((request, channel))) = self.rx.poll_recv(cx) { + let cid = request.into_cid(); let query_id = self.inner.get(&cid); - self.requests.insert(query_id, channel); + self.requests.insert(query_id, (request, channel)); } if let Poll::Ready(to_swarm) = self.inner.poll(cx) { diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index d0f804a1701..72445f2495d 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -55,11 +55,11 @@ use crate::{ }; use anyhow::{Context, anyhow}; use ethexe_common::{ - Address, BlockHeader, ValidatorsVec, + Address, Announce, BlockHeader, HashOf, ValidatorsVec, db::ConfigStorageRO, ecdsa::PublicKey, injected::{AddressedInjectedTransaction, SignedCompactPromise}, - network::{BitswapHandle, SignedValidatorMessage, VerifiedValidatorMessage}, + network::{BitswapAnnouncesHandle, SignedValidatorMessage, VerifiedValidatorMessage}, }; use ethexe_db::Database; use futures::{Stream, future::Either, ready, stream::FusedStream}; @@ -78,10 +78,7 @@ use libp2p::{ }; #[cfg(test)] use libp2p_swarm_test::SwarmExt; -use std::{ - collections::HashSet, fmt::Write, num::NonZeroU32, pin::Pin, sync::Arc, task::Poll, - time::Duration, -}; +use std::{collections::HashSet, fmt::Write, pin::Pin, sync::Arc, task::Poll, time::Duration}; use validator::{list::ValidatorList, topic::ValidatorTopic}; /// Default listen port. @@ -99,13 +96,9 @@ const MAX_ESTABLISHED_OUTGOING_CONNECTIONS: u32 = 500; const MAX_PENDING_INCOMING_CONNECTIONS: u32 = 10; const MAX_PENDING_OUTGOING_CONNECTIONS: u32 = 10; -/// Hard cap for the amount of announces that can be returned in one db-sync -/// response. -pub const DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE: NonZeroU32 = NonZeroU32::new(1000).unwrap(); - -impl BitswapHandle for bitswap::Handle { - async fn request(&self, hash: H256) -> Vec { - bitswap::Handle::request(self, hash).await +impl BitswapAnnouncesHandle for bitswap::Handle { + async fn request(&self, hash: HashOf) -> Announce { + bitswap::Handle::request(self, hash).await.unwrap_announce() } } @@ -159,8 +152,6 @@ pub struct NetworkConfig { /// Whether private and local addresses are allowed in discovery and /// identify flows. pub allow_non_global_addresses: bool, - /// Upper bound for `Announces` db-sync responses served by this node. - pub max_chain_len_for_announces_response: NonZeroU32, } impl NetworkConfig { @@ -175,7 +166,6 @@ impl NetworkConfig { transport_type: TransportType::Default, router_address, allow_non_global_addresses: false, - max_chain_len_for_announces_response: DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, } } @@ -189,7 +179,6 @@ impl NetworkConfig { transport_type: TransportType::Test, router_address, allow_non_global_addresses: true, - max_chain_len_for_announces_response: DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, } } } @@ -270,7 +259,6 @@ impl NetworkService { transport_type, router_address, allow_non_global_addresses, - max_chain_len_for_announces_response: _, } = config; let NetworkRuntimeConfig { @@ -947,7 +935,13 @@ mod tests { ) .await .expect("time has elapsed"); - assert_eq!(response, (b"hello".to_vec(), b"world".to_vec())); + assert_eq!( + response, + ( + bitswap::Response::Hash(b"hello".to_vec()), + bitswap::Response::Hash(b"world".to_vec()) + ) + ); } #[tokio::test] diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index ac1a18facf6..069b1e66ad3 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -31,7 +31,7 @@ use ethexe_common::{ router::{AnnouncesCommittedEvent, BatchCommittedEvent}, }, injected, - network::{AnnouncesRequest, AnnouncesRequestUntil, request_announces}, + network::{AnnouncesRequest, AnnouncesRequestUntil}, }; use ethexe_compute::ComputeService; use ethexe_db::{ @@ -44,7 +44,7 @@ use ethexe_db::{ visitor::DatabaseVisitor, }; use ethexe_ethereum::mirror::MirrorQuery; -use ethexe_network::{DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, NetworkService}; +use ethexe_network::NetworkService; use ethexe_observer::{ ObserverService, utils::{BlockId, BlockLoader}, @@ -136,7 +136,7 @@ async fn bitswap_fetch(network: &mut NetworkService, hash: H256) -> Vec { loop { tokio::select! { _ = network.select_next_some() => {}, - data = &mut fut => break data, + data = &mut fut => break data.unwrap_hash(), } } } @@ -146,17 +146,13 @@ async fn bitswap_fetch_announces( request: AnnouncesRequest, ) -> Result { let bitswap = network.bitswap_handle(); - let fut = request_announces( - &bitswap, - request, - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - ); + let fut = request.request(bitswap); tokio::pin!(fut); loop { tokio::select! { _ = network.select_next_some() => {}, - response = &mut fut => break response.map_err(Into::into), + response = &mut fut => break Ok(response), } } } @@ -184,7 +180,7 @@ async fn collect_announce( return Ok(announce); } - let response = bitswap_fetch_announces( + let mut response = bitswap_fetch_announces( network, AnnouncesRequest { head: announce_hash, @@ -194,8 +190,7 @@ async fn collect_announce( .await?; // Response is checked so we can just take the first announce - let (_, mut announces) = response.into_parts(); - Ok(announces.remove(0)) + Ok(response.announces.remove(0)) } /// Collects a set of valid code IDs that are not yet validated in the local database. diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 954a0e43b77..7574836453e 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -56,7 +56,7 @@ use anyhow::{Context, Result, bail}; use ethexe_blob_loader::{BlobLoader, BlobLoaderEvent, BlobLoaderService, ConsensusLayerConfig}; use ethexe_common::{ COMMITMENT_DELAY_LIMIT, CodeAndIdUnchecked, PromiseEmissionMode, - network::{AnnouncesRequestError, VerifiedValidatorMessage, request_announces}, + network::{AnnouncesResponse, VerifiedValidatorMessage}, }; use ethexe_compute::{ComputeConfig, ComputeEvent, ComputeService}; use ethexe_consensus::{ @@ -66,10 +66,7 @@ use ethexe_db::{ Database, GenesisInitializer, InitConfig, RawDatabase, RocksDatabase, dump::StateDump, }; use ethexe_ethereum::{EthereumBuilder, deploy::EthereumDeployer, router::RouterQuery}; -use ethexe_network::{ - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, NetworkEvent, NetworkRuntimeConfig, - NetworkService, -}; +use ethexe_network::{NetworkEvent, NetworkRuntimeConfig, NetworkService}; use ethexe_observer::{ ObserverConfig, ObserverEvent, ObserverService, utils::{BlockId, BlockLoader}, @@ -99,7 +96,7 @@ pub enum Event { BlobLoader(BlobLoaderEvent), Rpc(RpcEvent), Prometheus(PrometheusEvent), - Fetching(Result), + Fetching(AnnouncesResponse), } /// ethexe service. @@ -747,14 +744,7 @@ impl Service { }; let bitswap = network.bitswap_handle(); - network_fetcher.push(async move { - request_announces( - &bitswap, - request, - DEFAULT_MAX_CHAIN_LEN_FOR_ANNOUNCES_RESPONSE, - ) - .await - }); + network_fetcher.push(request.request(bitswap)); } ConsensusEvent::AnnounceAccepted(_) | ConsensusEvent::AnnounceRejected(_) => { // TODO #4940: consider to publish network message @@ -777,7 +767,7 @@ impl Service { unreachable!("Fetching event is impossible without network service"); }; - consensus.receive_announces_response(result?)?; + consensus.receive_announces_response(result)?; } } } diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index 80fc69ed17b..e4ab7cabd8a 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -17,7 +17,7 @@ // along with this program. If not, see . use crate::{ - RouterDataProvider, Service, + Service, config::EthereumConfig, tests::utils::{ InfiniteStreamExt, TestingEvent, TestingNetworkEvent, @@ -325,7 +325,6 @@ impl TestEnv { validator_key: None, general_signer: signer.clone(), network_signer: signer.clone(), - external_data_provider: Box::new(RouterDataProvider(router_query.clone())), db: db.clone(), }; @@ -1182,7 +1181,6 @@ impl Node { validator_key: self.validator_config.as_ref().map(|c| c.public_key), general_signer: self.signer.clone(), network_signer: self.signer.clone(), - external_data_provider: Box::new(RouterDataProvider(self.router_query.clone())), db: self.db.clone(), }; From 9313ad707ad630d771350cdd8ee836b13662fe42 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 14 May 2026 02:54:20 +0300 Subject: [PATCH 04/72] Initial fast sync --- ethexe/ethereum/src/lib.rs | 25 +++++++++- ethexe/ethereum/src/router/events.rs | 68 +++++++++++++++++++------- ethexe/observer/src/utils.rs | 32 +++--------- ethexe/service/src/fast_sync.rs | 70 +++++++++++++++++---------- ethexe/service/src/lib.rs | 10 ++-- ethexe/service/src/tests/utils/env.rs | 11 +++-- 6 files changed, 135 insertions(+), 81 deletions(-) diff --git a/ethexe/ethereum/src/lib.rs b/ethexe/ethereum/src/lib.rs index 11f2a8b26d9..e49e63bf3d7 100644 --- a/ethexe/ethereum/src/lib.rs +++ b/ethexe/ethereum/src/lib.rs @@ -27,7 +27,7 @@ use crate::{ }; use alloy::{ consensus::SignableTransaction, - eips::BlockId, + eips::{BlockId, BlockNumberOrTag}, network::{self, Ethereum as AlloyEthereum, EthereumWallet, Network, TxSigner}, primitives::{Address as AlloyAddress, B256, ChainId, Signature, U256 as AlloyU256, address}, providers::{ @@ -664,6 +664,29 @@ pub(crate) use signatures_consts; use crate::wvara::WVara; +/// A helping trait for converting various types into `alloy::eips::BlockNumberOrTag`. +pub trait IntoBlockNumberOrTag { + fn into_block_number_or_tag(self) -> BlockNumberOrTag; +} + +impl IntoBlockNumberOrTag for u32 { + fn into_block_number_or_tag(self) -> BlockNumberOrTag { + BlockNumberOrTag::Number(self as u64) + } +} + +impl IntoBlockNumberOrTag for u64 { + fn into_block_number_or_tag(self) -> BlockNumberOrTag { + BlockNumberOrTag::Number(self) + } +} + +impl IntoBlockNumberOrTag for BlockNumberOrTag { + fn into_block_number_or_tag(self) -> BlockNumberOrTag { + self + } +} + /// A helping trait for converting various types into `alloy::eips::BlockId`. pub trait IntoBlockId { fn into_block_id(self) -> BlockId; diff --git a/ethexe/ethereum/src/router/events.rs b/ethexe/ethereum/src/router/events.rs index be163671a50..cb4e5fc82ec 100644 --- a/ethexe/ethereum/src/router/events.rs +++ b/ethexe/ethereum/src/router/events.rs @@ -17,7 +17,7 @@ // along with this program. If not, see . use crate::{ - IRouter, + IRouter, IntoBlockNumberOrTag, abi::utils::{bytes32_to_code_id, bytes32_to_h256}, decode_log, router::RouterQuery, @@ -210,31 +210,47 @@ impl<'a> AnnouncesCommittedEventBuilder<'a> { pub struct CodeGotValidatedEventBuilder<'a> { event: Event<&'a RootProvider, IRouter::CodeGotValidated>, - valid: Option, } impl<'a> CodeGotValidatedEventBuilder<'a> { pub(crate) fn new(query: &'a RouterQuery) -> Self { Self { event: query.instance.CodeGotValidated_filter(), - valid: None, } } pub fn valid(mut self, valid: bool) -> Self { - self.valid = Some(valid); + self.event = self.event.topic1(valid); self } + pub fn from_block(mut self, block: impl IntoBlockNumberOrTag) -> Self { + self.event = self.event.from_block(block.into_block_number_or_tag()); + self + } + + pub fn to_block(mut self, block: impl IntoBlockNumberOrTag) -> Self { + self.event = self.event.to_block(block.into_block_number_or_tag()); + self + } + + pub async fn query(self) -> Result> { + Ok(self + .event + .chunked() + .query() + .await? + .into_iter() + .map(|(event, log)| (event.into(), log)) + .collect()) + } + pub async fn subscribe( self, ) -> Result> + Unpin + use<>> { - let mut event = self.event; - if let Some(valid) = self.valid { - event = event.topic1(valid); - } - Ok(event + Ok(self + .event .subscribe() .await? .into_stream() @@ -329,31 +345,47 @@ impl<'a> ComputationSettingsChangedEventBuilder<'a> { pub struct ProgramCreatedEventBuilder<'a> { event: Event<&'a RootProvider, IRouter::ProgramCreated>, - code_id: Option, } impl<'a> ProgramCreatedEventBuilder<'a> { pub(crate) fn new(query: &'a RouterQuery) -> Self { Self { event: query.instance.ProgramCreated_filter(), - code_id: None, } } pub fn code_id(mut self, code_id: CodeId) -> Self { - self.code_id = Some(code_id); + let code_id: B256 = code_id.into_bytes().into(); + self.event = self.event.topic1(code_id); self } + pub fn from_block(mut self, block: impl IntoBlockNumberOrTag) -> Self { + self.event = self.event.from_block(block.into_block_number_or_tag()); + self + } + + pub fn to_block(mut self, block: impl IntoBlockNumberOrTag) -> Self { + self.event = self.event.to_block(block.into_block_number_or_tag()); + self + } + + pub async fn query(self) -> Result> { + Ok(self + .event + .chunked() + .query() + .await? + .into_iter() + .map(|(event, log)| (event.into(), log)) + .collect()) + } + pub async fn subscribe( self, ) -> Result> + Unpin + use<>> { - let mut event = self.event; - if let Some(code_id) = self.code_id { - let code_id: B256 = code_id.into_bytes().into(); - event = event.topic1(code_id); - } - Ok(event + Ok(self + .event .subscribe() .await? .into_stream() diff --git a/ethexe/observer/src/utils.rs b/ethexe/observer/src/utils.rs index f1a96ac997c..70e78f4b69e 100644 --- a/ethexe/observer/src/utils.rs +++ b/ethexe/observer/src/utils.rs @@ -32,7 +32,7 @@ use alloy::{ }; use anyhow::{Context, Result}; use ethexe_common::{Address, BlockData, BlockHeader, SimpleBlockData, events::BlockEvent}; -use ethexe_ethereum::{abi::IRouter, mirror, router}; +use ethexe_ethereum::{IntoBlockId, abi::IRouter, mirror, router}; use futures::{TryFutureExt, future}; use gprimitives::H256; use std::{collections::HashMap, future::IntoFuture, ops::RangeInclusive}; @@ -45,26 +45,9 @@ const LOGS_CHUNK_SIZE: u64 = 256; /// Maximum number of in-flight log chunk requests issued by [`alloy::contract::ChunkedEvent`]. const LOGS_MAX_CONCURRENCY: usize = 8; -#[derive(Debug, Copy, Clone, PartialEq, Eq, derive_more::From)] -pub enum BlockId { - Hash(H256), - Latest, - Finalized, -} - -impl BlockId { - fn as_alloy(self) -> alloy::eips::BlockId { - match self { - BlockId::Hash(hash) => alloy::eips::BlockId::hash(hash.0.into()), - BlockId::Latest => alloy::eips::BlockId::latest(), - BlockId::Finalized => alloy::eips::BlockId::finalized(), - } - } -} - #[allow(async_fn_in_trait)] pub trait BlockLoader { - async fn load_simple(&self, block: BlockId) -> Result; + async fn load_simple(&self, block: impl IntoBlockId) -> Result; async fn load(&self, block: H256, header: Option) -> Result; @@ -210,13 +193,10 @@ impl EthereumBlockLoader { } impl BlockLoader for EthereumBlockLoader { - async fn load_simple(&self, block: BlockId) -> Result { + async fn load_simple(&self, block: impl IntoBlockId) -> Result { + let block = block.into_block_id(); log::trace!("Querying simple data for one block {block:?}"); - let block = self - .provider - .get_block(block.as_alloy()) - .into_future() - .await?; + let block = self.provider.get_block(block).into_future().await?; let block = block.context("block not found")?; let (hash, header) = Self::block_response_to_data(block); Ok(SimpleBlockData { hash, header }) @@ -232,7 +212,7 @@ impl BlockLoader for EthereumBlockLoader { let (block_hash, header, logs) = if let Some(header) = header { (block, header, logs_request.await?) } else { - let data = self.load_simple(block.into()); + let data = self.load_simple(block); let (SimpleBlockData { hash, header }, logs) = future::try_join(data, logs_request).await?; (hash, header, logs) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 069b1e66ad3..1ee1cf08a02 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -17,6 +17,7 @@ // along with this program. If not, see . use crate::Service; +use alloy::eips::BlockId; use anyhow::{Context, Result}; use ethexe_common::{ Address, Announce, BlockData, CodeAndIdUnchecked, Digest, HashOf, ProgramStates, @@ -45,10 +46,7 @@ use ethexe_db::{ }; use ethexe_ethereum::mirror::MirrorQuery; use ethexe_network::NetworkService; -use ethexe_observer::{ - ObserverService, - utils::{BlockId, BlockLoader}, -}; +use ethexe_observer::{ObserverService, utils::BlockLoader}; use ethexe_runtime_common::{ ScheduleRestorer, state::{ @@ -160,15 +158,21 @@ async fn bitswap_fetch_announces( /// Collects program code IDs for the latest committed block. async fn collect_program_code_ids( observer: &mut ObserverService, - _network: &mut NetworkService, - latest_committed_block: H256, + genesis_block: u32, + latest_committed_block: u32, ) -> Result> { - let router_query = observer.router_query(); - let _programs_count = router_query - .programs_count_at(latest_committed_block) - .await?; - - anyhow::bail!("fast-sync program id enumeration via bitswap is not implemented yet") + Ok(observer + .router_query() + .events() + .program_created() + .from_block(genesis_block) + .to_block(latest_committed_block) + .query() + .await + .context("failed to ProgramCreated events")? + .into_iter() + .map(|(event, _log)| (event.actor_id, event.code_id)) + .collect()) } async fn collect_announce( @@ -196,17 +200,25 @@ async fn collect_announce( /// Collects a set of valid code IDs that are not yet validated in the local database. async fn collect_code_ids( observer: &mut ObserverService, - _network: &mut NetworkService, - db: &Database, - latest_committed_block: H256, + genesis_block: u32, + latest_committed_block: u32, ) -> Result> { - let router_query = observer.router_query(); - let _codes_count = router_query - .validated_codes_count_at(latest_committed_block) - .await?; - - let _ = db; - anyhow::bail!("fast-sync valid code enumeration via bitswap is not implemented yet") + Ok(observer + .router_query() + .events() + .code_got_validated() + .valid(true) + .from_block(genesis_block) + .to_block(latest_committed_block) + .query() + .await + .context("failed to query CodeGotValidated events")? + .into_iter() + .map(|(event, _log)| { + debug_assert!(event.valid); + event.code_id + }) + .collect()) } /// Collects the program states for a given set of program IDs at a specified block height. @@ -647,11 +659,19 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { // we get finalized block to avoid block reorganization // because we restore the database only for the latest block of a chain, // and thus the reorganization can lead us to an empty block - .load_simple(BlockId::Finalized) + .load_simple(BlockId::finalized()) .await .context("failed to get latest block")? .hash; + let genesis_block = db.config().genesis_block_hash; + let genesis_block = observer + .block_loader() + .load_simple(genesis_block) + .await + .context("failed to get genesis block")?; + let genesis_block = genesis_block.header.height; + let block_loader = observer.block_loader(); let Some(EventData { @@ -676,8 +696,8 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { events, } = block_loader.load(announce.block_hash, None).await?; - let code_ids = collect_code_ids(observer, network, db, announce.block_hash).await?; - let program_code_ids = collect_program_code_ids(observer, network, announce.block_hash).await?; + let code_ids = collect_code_ids(observer, genesis_block, header.height).await?; + let program_code_ids = collect_program_code_ids(observer, genesis_block, header.height).await?; // we fetch program states from the finalized block // because actual states are at the same block as we acquired the latest committed block let program_states = diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 7574836453e..7221e59eb8d 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -48,6 +48,7 @@ use crate::config::{Config, ConfigPublicKey}; use alloy::{ + eips::BlockId, node_bindings::{Anvil, AnvilInstance}, providers::{ProviderBuilder, RootProvider, ext::AnvilApi}, rpc::types::anvil::Metadata, @@ -67,10 +68,7 @@ use ethexe_db::{ }; use ethexe_ethereum::{EthereumBuilder, deploy::EthereumDeployer, router::RouterQuery}; use ethexe_network::{NetworkEvent, NetworkRuntimeConfig, NetworkService}; -use ethexe_observer::{ - ObserverConfig, ObserverEvent, ObserverService, - utils::{BlockId, BlockLoader}, -}; +use ethexe_observer::{ObserverConfig, ObserverEvent, ObserverService, utils::BlockLoader}; use ethexe_processor::{ProcessedCodeInfo, Processor, ProcessorConfig, ValidCodeInfo}; use ethexe_prometheus::{PrometheusEvent, PrometheusService}; use ethexe_rpc::{RpcEvent, RpcServer}; @@ -301,7 +299,7 @@ impl Service { let latest_block = observer .block_loader() - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .context("failed to get latest block")?; @@ -404,7 +402,7 @@ impl Service { let latest_block_data = observer .block_loader() - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .context("failed to get latest block")?; diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index e4ab7cabd8a..643bdbfff90 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -25,6 +25,7 @@ use crate::{ }, }; use alloy::{ + eips::BlockId, node_bindings::{Anvil, AnvilInstance}, providers::{ProviderBuilder, RootProvider, ext::AnvilApi}, rpc::types::anvil::{Metadata, MineOptions}, @@ -56,7 +57,7 @@ use ethexe_ethereum::{ use ethexe_network::{NetworkConfig, NetworkRuntimeConfig, NetworkService, export::Multiaddr}; use ethexe_observer::{ ObserverConfig, ObserverService, - utils::{BlockId, BlockLoader, EthereumBlockLoader}, + utils::{BlockLoader, EthereumBlockLoader}, }; use ethexe_processor::{DEFAULT_CHUNK_SIZE, Processor}; use ethexe_rpc::{DEFAULT_BLOCK_GAS_LIMIT_MULTIPLIER, RpcConfig, RpcServer}; @@ -268,7 +269,7 @@ impl TestEnv { .unwrap(); let latest_block = observer .block_loader() - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .context("failed to get latest block")?; let latest_validators = router_query @@ -677,7 +678,7 @@ impl TestEnv { pub async fn latest_block(&self) -> SimpleBlockData { EthereumBlockLoader::new(self.provider.clone(), self.eth_cfg.router_address) - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .unwrap() } @@ -975,7 +976,7 @@ impl Node { .unwrap(); let latest_block = observer .block_loader() - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .unwrap(); let latest_validators = observer @@ -1203,7 +1204,7 @@ impl Node { let provider = RootProvider::connect(&self.eth_cfg.rpc).await.unwrap(); let block_loader = EthereumBlockLoader::new(provider, self.eth_cfg.router_address); - let latest_block = block_loader.load_simple(BlockId::Latest).await.unwrap(); + let latest_block = block_loader.load_simple(BlockId::latest()).await.unwrap(); let latest_validators = self .router_query .validators_at(latest_block.hash) From 7abbff0d0d4fafcd981dff81311875da384c79ad Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 14 May 2026 03:13:19 +0300 Subject: [PATCH 05/72] Remove unused deps --- Cargo.lock | 4 ---- ethexe/network/Cargo.toml | 3 --- ethexe/observer/Cargo.toml | 1 - 3 files changed, 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f3509512cee..12dd43c7aaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5392,14 +5392,12 @@ dependencies = [ "derive_more 2.1.1", "ethexe-common", "ethexe-db", - "ethexe-service-utils", "futures", "gear-workspace-hack", "gprimitives", "gsigner", "indexmap 2.13.0", "ip_network", - "itertools 0.13.0", "libp2p 0.56.0", "libp2p-gossipsub", "libp2p-swarm-test", @@ -5413,7 +5411,6 @@ dependencies = [ "prometheus-client 0.23.1", "proptest", "rand 0.8.5", - "thiserror 2.0.17", "tokio", "tracing-subscriber", ] @@ -5465,7 +5462,6 @@ version = "1.10.0" dependencies = [ "alloy", "anyhow", - "derive_more 2.1.1", "ethexe-common", "ethexe-db", "ethexe-ethereum", diff --git a/ethexe/network/Cargo.toml b/ethexe/network/Cargo.toml index f9e18effc68..d9d160e6280 100644 --- a/ethexe/network/Cargo.toml +++ b/ethexe/network/Cargo.toml @@ -17,7 +17,6 @@ ignored = ["libp2p-gossipsub"] # workspace members gsigner = { workspace = true, features = ["std", "secp256k1", "codec", "keyring", "serde"] } ethexe-db.workspace = true -ethexe-service-utils.workspace = true ethexe-common.workspace = true gprimitives = { workspace = true, features = ["std", "codec"] } @@ -38,11 +37,9 @@ async-trait.workspace = true rand = { workspace = true, features = ["std", "std_rng"] } futures.workspace = true derive_more.workspace = true -itertools = { workspace = true, features = ["use_std"] } nonempty.workspace = true auto_impl.workspace = true lru.workspace = true -thiserror.workspace = true indexmap.workspace = true ip_network.workspace = true prometheus-client = "0.23.1" # specific version that lip2p uses diff --git a/ethexe/observer/Cargo.toml b/ethexe/observer/Cargo.toml index 57865de136f..0c31b97d31b 100644 --- a/ethexe/observer/Cargo.toml +++ b/ethexe/observer/Cargo.toml @@ -33,7 +33,6 @@ tokio = { workspace = true, features = ["rt-multi-thread", "fs", "sync"] } futures.workspace = true log.workspace = true tracing.workspace = true -derive_more.workspace = true future-timing.workspace = true gear-workspace-hack.workspace = true From 1b22b0fbfaf789041c27bf87750c82a7dc4c8b05 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 14 May 2026 14:26:58 +0300 Subject: [PATCH 06/72] Update docs --- ethexe/network/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index 72445f2495d..2d656b77e82 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -24,14 +24,14 @@ //! - peer management and connection caps; //! - Kademlia-backed validator discovery; //! - gossipsub topics for validator messages and public promises; -//! - request/response database synchronization; +//! - bitswap-backed data fetching; //! - private injected-transaction delivery to validators; //! - peer scoring and temporary peer blocking. //! //! [`NetworkService`] is the main integration point used by higher-level //! services. It owns the swarm, emits validated [`NetworkEvent`] items, and -//! hands out protocol-specific handles such as [`db_sync::Handle`] for -//! database synchronization and a peer-scoring handle for internal use. +//! hands out protocol-specific handles such as the bitswap handle for data +//! fetching and a peer-scoring handle for internal use. mod bitswap; mod gossipsub; @@ -195,7 +195,7 @@ pub struct NetworkRuntimeConfig { pub general_signer: Signer, /// Signer used only to construct the libp2p networking keypair. pub network_signer: Signer, - /// Database backing validator discovery and db-sync responses. + /// Database backing validator discovery and bitswap responses. pub db: Database, } From 5a8c5840ad846a58cff2495930f6b404a52e367d Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 14 May 2026 15:01:32 +0300 Subject: [PATCH 07/72] Add tests --- ethexe/network/src/bitswap.rs | 235 +++++++++++++++++++++++++++++++--- ethexe/network/src/lib.rs | 10 +- 2 files changed, 223 insertions(+), 22 deletions(-) diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs index cb4057252f5..8c3fd44d689 100644 --- a/ethexe/network/src/bitswap.rs +++ b/ethexe/network/src/bitswap.rs @@ -40,12 +40,17 @@ use std::{ mem, sync::Arc, task::{Context, Poll}, + time::Duration, }; use tokio::{ sync::{mpsc, oneshot}, - task, + task, time, }; +const BLAKE2B_CODE: u64 = 0xb220; // standard BLAKE2b multihash code +const RAW_CODEC: u64 = 0x55; // standard CID raw codec +const ANNOUNCES_CODEC: u64 = 0x300000; // user-defined CID codec + #[derive(Debug, Copy, Clone, Eq, PartialEq, derive_more::From)] pub enum Request { Hash(H256), @@ -56,13 +61,12 @@ impl Request { fn into_cid(self) -> Cid { match self { Request::Hash(hash) => Cid::new_v1( - Blockstore::RAW_CODEC, - Multihash::wrap(Blockstore::BLAKE2B_CODE, hash.as_bytes()) - .expect("size is always correct"), + RAW_CODEC, + Multihash::wrap(BLAKE2B_CODE, hash.as_bytes()).expect("size is always correct"), ), Request::Announce(hash) => Cid::new_v1( - Blockstore::ANNOUNCES_CODEC, - Multihash::wrap(Blockstore::BLAKE2B_CODE, hash.inner().as_bytes()) + ANNOUNCES_CODEC, + Multihash::wrap(BLAKE2B_CODE, hash.inner().as_bytes()) .expect("size is always correct"), ), } @@ -84,15 +88,56 @@ pub enum Response { Announce(Announce), } +#[derive(Debug, Clone, Copy, Default)] +pub struct Config { + /// Restart stalled requests after some time. + /// + /// This is intended for test environment only, where a peer + /// can receive an announce request before it has caught up enough to serve + /// the corresponding data. Production environment is expected to have + /// enough peers to fulfill requests, so request scheduling is left to + /// Bitswap itself. + pub auto_retry: bool, +} + +impl Config { + pub fn with_auto_retry(mut self, auto_retry: bool) -> Self { + self.auto_retry = auto_retry; + self + } +} + #[derive(Clone)] -pub struct Handle(mpsc::UnboundedSender<(Request, oneshot::Sender)>); +pub struct Handle { + inner: mpsc::UnboundedSender<(Request, oneshot::Sender)>, + auto_retry: bool, +} impl Handle { + const RETRY_TIMEOUT: Duration = Duration::from_secs(5); + pub async fn request(&self, request: impl Into) -> Response { + let request = request.into(); + + if !self.auto_retry { + return self.inner_request(request).await; + } + + loop { + match time::timeout(Self::RETRY_TIMEOUT, self.inner_request(request)).await { + Ok(response) => return response, + Err(_) => { + log::warn!("Bitswap request {request:?} timed out, retrying"); + } + } + } + } + + async fn inner_request(&self, request: Request) -> Response { let (tx, rx) = oneshot::channel(); - self.0 - .send((request.into(), tx)) + self.inner + .send((request, tx)) .expect("channel should never be closed"); rx.await.expect("channel should never be closed") @@ -117,17 +162,14 @@ pub struct Blockstore { impl Blockstore { const MAX_BLOCK_SIZE: u64 = 1024 * 1024; // 1MB - const BLAKE2B_CODE: u64 = 0xb220; - const RAW_CODEC: u64 = 0x55; - const ANNOUNCES_CODEC: u64 = 0x300000; fn convert_multihash(multihash: &Multihash) -> blockstore::Result { let hash: Multihash<32> = beetswap::utils::convert_multihash(multihash).ok_or(blockstore::Error::CidTooLarge)?; - if hash.code() != Self::BLAKE2B_CODE { + if hash.code() != BLAKE2B_CODE { return Err(blockstore::Error::CidError(CidError::InvalidMultihashCode( hash.code(), - Self::BLAKE2B_CODE, + BLAKE2B_CODE, ))); } if hash.size() as usize != mem::size_of::() { @@ -150,7 +192,7 @@ impl blockstore::Blockstore for Blockstore { task::spawn_blocking(move || { let hash = Self::convert_multihash(&hash)?; match codec { - Self::RAW_CODEC => { + RAW_CODEC => { let data = db.read_by_hash(hash); if let Some(data) = &data @@ -162,7 +204,7 @@ impl blockstore::Blockstore for Blockstore { Ok(data) } - Self::ANNOUNCES_CODEC => { + ANNOUNCES_CODEC => { let hash = unsafe { HashOf::new(hash) }; let announce = db.announce(hash); @@ -208,13 +250,12 @@ impl Multihasher<32> for Blake2b256Multihasher { multihash_code: u64, input: &[u8], ) -> Result, MultihasherError> { - if multihash_code != Blockstore::BLAKE2B_CODE { + if multihash_code != BLAKE2B_CODE { return Err(MultihasherError::UnknownMultihashCode); } let hash = ethexe_db::hash(input); - let hash = Multihash::wrap(Blockstore::BLAKE2B_CODE, hash.as_bytes()) - .expect("size is always correct"); + let hash = Multihash::wrap(BLAKE2B_CODE, hash.as_bytes()).expect("size is always correct"); Ok(hash) } } @@ -229,7 +270,7 @@ pub struct Behaviour { } impl Behaviour { - pub fn new(db: Box) -> Self { + pub fn new(db: Box, config: Config) -> Self { let (handle, rx) = mpsc::unbounded_channel(); let blockstore = Arc::new(Blockstore { db }); @@ -239,7 +280,10 @@ impl Behaviour { .protocol_prefix("/ethexe") .expect("prefix is always correct") .build(), - handle: Handle(handle), + handle: Handle { + inner: handle, + auto_retry: config.auto_retry, + }, rx, requests: HashMap::new(), } @@ -377,3 +421,152 @@ impl NetworkBehaviour for Behaviour { Poll::Pending } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::tests::arb_value; + use assert_matches::assert_matches; + use blockstore::Blockstore as _; + use ethexe_common::db::AnnounceStorageRW; + + #[test] + fn request_converts_to_expected_cid() { + let hash = H256::from([1; 32]); + let cid = Request::Hash(hash).into_cid(); + assert_eq!(cid.codec(), RAW_CODEC); + assert_eq!(cid.hash().code(), BLAKE2B_CODE); + assert_eq!(cid.hash().digest(), hash.as_bytes()); + + let announce_hash = unsafe { HashOf::new(H256::from([2; 32])) }; + let cid = Request::Announce(announce_hash).into_cid(); + assert_eq!(cid.codec(), ANNOUNCES_CODEC); + assert_eq!(cid.hash().code(), BLAKE2B_CODE); + assert_eq!(cid.hash().digest(), announce_hash.inner().as_bytes()); + } + + #[test] + fn announce_request_decodes_response() { + let announce = arb_value::(()); + let response = Request::Announce(announce.to_hash()).into_response(announce.encode()); + + assert_eq!(response, Response::Announce(announce)); + } + + #[tokio::test] + async fn blockstore_reads_raw_data() { + let db = ethexe_db::Database::memory(); + let hash = db.cas().write(b"hello"); + let blockstore = Blockstore { db: Box::new(db) }; + let cid = Request::Hash(hash).into_cid(); + + let data = blockstore.get(&cid).await.unwrap(); + + assert_eq!(data, Some(b"hello".to_vec())); + } + + #[tokio::test] + async fn blockstore_reads_announces() { + let db = ethexe_db::Database::memory(); + let announce = arb_value::(()); + let announce_hash = db.set_announce(announce.clone()); + let blockstore = Blockstore { db: Box::new(db) }; + let cid = Request::Announce(announce_hash).into_cid(); + + let data = blockstore.get(&cid).await.unwrap(); + + assert_eq!(data, Some(announce.encode())); + } + + #[tokio::test] + async fn blockstore_rejects_unknown_codec() { + let db = ethexe_db::Database::memory(); + let blockstore = Blockstore { db: Box::new(db) }; + let hash = H256::from([3; 32]); + let multihash = Multihash::wrap(BLAKE2B_CODE, hash.as_bytes()).unwrap(); + let cid = Cid::new_v1(0x99, multihash); + + let error = blockstore.get(&cid).await.unwrap_err(); + + assert_matches!( + error, + blockstore::Error::CidError(CidError::InvalidCidCodec(0x99)) + ); + } + + #[tokio::test] + async fn blockstore_rejects_unknown_multihash_code() { + let db = ethexe_db::Database::memory(); + let blockstore = Blockstore { db: Box::new(db) }; + let hash = H256::from([4; 32]); + let multihash = Multihash::wrap(0x12, hash.as_bytes()).unwrap(); + let cid = Cid::new_v1(RAW_CODEC, multihash); + + let error = blockstore.get(&cid).await.unwrap_err(); + + assert_matches!( + error, + blockstore::Error::CidError(CidError::InvalidMultihashCode(0x12, BLAKE2B_CODE)) + ); + } + + #[tokio::test] + async fn blockstore_rejects_oversized_raw_data() { + let db = ethexe_db::Database::memory(); + let hash = db.cas().write(&vec![0; MAX_BLOCK_SIZE as usize + 1]); + let blockstore = Blockstore { db: Box::new(db) }; + let cid = Request::Hash(hash).into_cid(); + + let error = blockstore.get(&cid).await.unwrap_err(); + + assert_matches!(error, blockstore::Error::ValueTooLarge); + } + + #[tokio::test] + async fn blake2b_multihasher_hashes_known_code() { + let multihash = Blake2b256Multihasher + .hash(BLAKE2B_CODE, b"hello") + .await + .unwrap(); + + assert_eq!(multihash.code(), BLAKE2B_CODE); + assert_eq!(multihash.digest(), ethexe_db::hash(b"hello").as_bytes()); + } + + #[tokio::test] + async fn blake2b_multihasher_rejects_unknown_code() { + let error = Blake2b256Multihasher + .hash(0x12, b"hello") + .await + .unwrap_err(); + + assert_matches!(error, MultihasherError::UnknownMultihashCode); + } + + #[tokio::test(start_paused = true)] + async fn handle_retries_timed_out_requests() { + let (inner, mut rx) = mpsc::unbounded_channel(); + let handle = Handle { + inner, + auto_retry: true, + }; + let hash = H256::from([5; 32]); + + let pending = tokio::spawn(async move { handle.request(hash).await }); + + let (request, first_response) = rx.recv().await.unwrap(); + assert_eq!(request, Request::Hash(hash)); + + time::advance(Handle::RETRY_TIMEOUT).await; + let (request, second_response) = rx.recv().await.unwrap(); + assert_eq!(request, Request::Hash(hash)); + assert!(first_response.is_closed()); + + second_response + .send(Response::Hash(b"hello".to_vec())) + .unwrap(); + let response = pending.await.unwrap(); + + assert_eq!(response, Response::Hash(b"hello".to_vec())); + } +} diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index 2d656b77e82..7c1e21fbf36 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -789,7 +789,15 @@ impl Behaviour { ) .map_err(|e| anyhow!("`gossipsub::Behaviour` error: {e}"))?; - let bitswap = bitswap::Behaviour::new(db); + // The test environment can start an announce request while the + // serving peer is still catching up and does not have the announce in + // its local DB yet. Retrying keeps those deterministic integration + // tests from waiting forever; the production environment is expected + // to have enough peers to fulfill requests, so scheduling is left to + // Bitswap itself. + let bitswap = bitswap::Config::default() + .with_auto_retry(matches!(transport_type, TransportType::Test)); + let bitswap = bitswap::Behaviour::new(db, bitswap); let injected = injected::Behaviour::new(); From 692b44714ee7e10a206c444cc4f1cd96bb16ee89 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 14 May 2026 15:11:07 +0300 Subject: [PATCH 08/72] Map database panics to `FatalDatabaseError` and add corresponding test. --- ethexe/network/src/bitswap.rs | 79 +++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs index 8c3fd44d689..cfdafe38e3b 100644 --- a/ethexe/network/src/bitswap.rs +++ b/ethexe/network/src/bitswap.rs @@ -222,7 +222,10 @@ impl blockstore::Blockstore for Blockstore { ))), } }) - .map(|res| res.expect("database panicked")) + .map(|res| { + res.map_err(|err| blockstore::Error::FatalDatabaseError(err.to_string())) + .flatten() + }) } async fn put_keyed( @@ -428,7 +431,59 @@ mod tests { use crate::utils::tests::arb_value; use assert_matches::assert_matches; use blockstore::Blockstore as _; - use ethexe_common::db::AnnounceStorageRW; + use ethexe_common::{ + ProgramStates, Schedule, + db::{AnnounceMeta, AnnounceStorageRW}, + gear::StateTransition, + }; + use std::collections::BTreeSet; + + #[derive(Clone)] + struct PanickingDatabase; + + impl HashStorageRO for PanickingDatabase { + fn read_by_hash(&self, _hash: H256) -> Option> { + panic!("database read panic"); + } + } + + impl AnnounceStorageRO for PanickingDatabase { + fn announce(&self, _hash: HashOf) -> Option { + panic!("database announce panic"); + } + + fn announce_program_states( + &self, + _announce_hash: HashOf, + ) -> Option { + unreachable!("not used in this test") + } + + fn announce_outcome( + &self, + _announce_hash: HashOf, + ) -> Option> { + unreachable!("not used in this test") + } + + fn announce_schedule(&self, _announce_hash: HashOf) -> Option { + unreachable!("not used in this test") + } + + fn announce_meta(&self, _announce_hash: HashOf) -> AnnounceMeta { + unreachable!("not used in this test") + } + + fn block_announces(&self, _block_hash: H256) -> Option>> { + unreachable!("not used in this test") + } + } + + impl BlockstoreDatabase for PanickingDatabase { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + } #[test] fn request_converts_to_expected_cid() { @@ -513,7 +568,9 @@ mod tests { #[tokio::test] async fn blockstore_rejects_oversized_raw_data() { let db = ethexe_db::Database::memory(); - let hash = db.cas().write(&vec![0; MAX_BLOCK_SIZE as usize + 1]); + let hash = db + .cas() + .write(&vec![0; Blockstore::MAX_BLOCK_SIZE as usize + 1]); let blockstore = Blockstore { db: Box::new(db) }; let cid = Request::Hash(hash).into_cid(); @@ -522,6 +579,22 @@ mod tests { assert_matches!(error, blockstore::Error::ValueTooLarge); } + #[tokio::test] + async fn blockstore_maps_database_panic_to_fatal_database_error() { + let blockstore = Blockstore { + db: Box::new(PanickingDatabase), + }; + let cid = Request::Hash(H256::from([6; 32])).into_cid(); + + let error = blockstore.get(&cid).await.unwrap_err(); + + assert_matches!( + error, + blockstore::Error::FatalDatabaseError(message) + if message.contains("database read panic") + ); + } + #[tokio::test] async fn blake2b_multihasher_hashes_known_code() { let multihash = Blake2b256Multihasher From 61241cc8a104b963b8678d52786a0cf63966510b Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Tue, 26 May 2026 00:18:19 +0300 Subject: [PATCH 09/72] Initial fast sync --- ethexe/network/src/bitswap.rs | 41 +- ethexe/network/src/lib.rs | 199 ++++++- ethexe/network/src/mb_sync.rs | 392 +++++++++++++ ethexe/service/src/fast_sync.rs | 784 +++++++++++++++++++++++++- ethexe/service/src/lib.rs | 12 +- ethexe/service/src/tests/mod.rs | 116 +++- ethexe/service/src/tests/utils/env.rs | 60 +- 7 files changed, 1552 insertions(+), 52 deletions(-) create mode 100644 ethexe/network/src/mb_sync.rs diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs index b90def1c030..2ba2940c042 100644 --- a/ethexe/network/src/bitswap.rs +++ b/ethexe/network/src/bitswap.rs @@ -46,19 +46,27 @@ use tokio::{ const BLAKE2B_CODE: u64 = 0xb220; // standard BLAKE2b multihash code const RAW_CODEC: u64 = 0x55; // standard CID raw codec -#[derive(Debug, Copy, Clone, Eq, PartialEq, derive_more::From)] +#[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum Request { Hash(H256), } +impl From for Request { + fn from(hash: H256) -> Self { + Self::Hash(hash) + } +} + impl Request { fn into_cid(self) -> Cid { - match self { - Request::Hash(hash) => Cid::new_v1( - RAW_CODEC, - Multihash::wrap(BLAKE2B_CODE, hash.as_bytes()).expect("size is always correct"), - ), - } + let (codec, hash) = match self { + Request::Hash(hash) => (RAW_CODEC, hash), + }; + + Cid::new_v1( + codec, + Multihash::wrap(BLAKE2B_CODE, hash.as_bytes()).expect("size is always correct"), + ) } fn into_response(self, data: Vec) -> Response { @@ -129,9 +137,7 @@ impl Handle { } } -pub(crate) trait BlockstoreDatabase: - Send + Sync + HashStorageRO -{ +pub(crate) trait BlockstoreDatabase: Send + Sync + HashStorageRO { fn clone_boxed(&self) -> Box; } @@ -176,7 +182,7 @@ impl blockstore::Blockstore for Blockstore { let codec = cid.codec(); task::spawn_blocking(move || { let hash = Self::convert_multihash(&hash)?; - match codec { + let data = match codec { RAW_CODEC => { let data = db.read_by_hash(hash); @@ -187,12 +193,21 @@ impl blockstore::Blockstore for Blockstore { return Err(blockstore::Error::ValueTooLarge); } - Ok(data) + data } codec => Err(blockstore::Error::CidError(CidError::InvalidCidCodec( codec, - ))), + )))?, + }; + + if let Some(data) = &data + && data.len() as u64 > Self::MAX_BLOCK_SIZE + { + log::warn!("{hash} is too large: {} bytes", data.len()); + return Err(blockstore::Error::ValueTooLarge); } + + Ok(data) }) .map(|res| { res.map_err(|err| blockstore::Error::FatalDatabaseError(err.to_string())) diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index 8e2150b97fe..4d55275e7f2 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -9,19 +9,21 @@ //! - peer management and connection caps; //! - Kademlia-backed validator discovery; //! - gossipsub topics for validator messages and public promises; -//! - bitswap-backed data fetching; +//! - bitswap-backed content fetching; +//! - request/response MB row fetching; //! - private injected-transaction delivery to validators; //! - peer scoring and temporary peer blocking. //! //! [`NetworkService`] is the main integration point used by higher-level //! services. It owns the swarm, emits validated [`NetworkEvent`] items, and -//! hands out protocol-specific handles such as the bitswap handle for data +//! hands out protocol-specific handles such as the bitswap handle for content //! fetching and a peer-scoring handle for internal use. mod bitswap; mod gossipsub; mod injected; mod kad; +mod mb_sync; mod peer_score; mod slots; mod utils; @@ -40,9 +42,10 @@ use crate::{ }; use anyhow::{Context, anyhow}; use ethexe_common::{ - Address, BlockHeader, ValidatorsVec, + Address, BlockHeader, ProgramStates, Schedule, ValidatorsVec, db::ConfigStorageRO, ecdsa::PublicKey, + gear::StateTransition, injected::{AddressedInjectedTransaction, SignedCompactPromise}, network::{SignedValidatorMessage, VerifiedValidatorMessage}, }; @@ -58,7 +61,10 @@ use libp2p::{ metrics::Recorder, multiaddr::Protocol, ping, - swarm::{Config as SwarmConfig, NetworkBehaviour, SwarmEvent, behaviour::toggle::Toggle}, + swarm::{ + Config as SwarmConfig, NetworkBehaviour, SwarmEvent, behaviour::toggle::Toggle, + dial_opts::DialOpts, + }, yamux, }; #[cfg(test)] @@ -269,6 +275,7 @@ impl NetworkService { router_address, keypair: keypair.clone(), db: bitswap::BlockstoreDatabase::clone_boxed(&db), + mb_sync_db: mb_sync::MbSyncDatabase::clone_boxed(&db), transport_type, validator_key, general_signer, @@ -308,8 +315,25 @@ impl NetworkService { }) .context("bootstrap nodes are not allowed without peer ID")?; - swarm.add_peer_address(peer_id, multiaddr.clone()); - swarm.behaviour_mut().kad.add_address(peer_id, multiaddr); + let mut peer_addr = Multiaddr::empty(); + for protocol in multiaddr.iter() { + if !matches!(protocol, Protocol::P2p(_)) { + peer_addr.push(protocol); + } + } + + swarm.add_peer_address(peer_id, peer_addr.clone()); + swarm + .behaviour_mut() + .kad + .add_address(peer_id, peer_addr.clone()); + if let Err(error) = swarm.dial( + DialOpts::peer_id(peer_id) + .addresses(vec![peer_addr]) + .build(), + ) { + log::warn!("failed to dial bootstrap peer {peer_id}: {error}"); + } bootstrap_peers.insert(peer_id); } @@ -419,6 +443,7 @@ impl NetworkService { BehaviourEvent::Kad(event) => self.handle_kad_event(event), BehaviourEvent::Gossipsub(event) => return self.handle_gossipsub_event(event), BehaviourEvent::Bitswap(_event) => {} + BehaviourEvent::MbSync(_event) => {} BehaviourEvent::Injected(event) => return self.handle_injected_event(event), BehaviourEvent::ValidatorDiscovery(event) => { return self.handle_validator_discovery_event(event); @@ -602,6 +627,93 @@ impl NetworkService { self.swarm.behaviour().bitswap.handle() } + /// Handle used by external services to start MB row requests. + pub fn mb_sync_handle(&self) -> mb_sync::Handle { + self.swarm.behaviour().mb_sync.handle() + } + + async fn bitswap_request(&mut self, request: bitswap::Request) -> bitswap::Response { + let bitswap = self.bitswap_handle(); + let fut = bitswap.request(request); + tokio::pin!(fut); + + loop { + tokio::select! { + _ = self.select_next_some() => {}, + response = &mut fut => break response, + } + } + } + + async fn mb_sync_request(&mut self, request: mb_sync::Request) -> mb_sync::Response { + let mb_sync = self.mb_sync_handle(); + let fut = mb_sync.request(request); + tokio::pin!(fut); + + loop { + tokio::select! { + _ = self.select_next_some() => {}, + response = &mut fut => break response, + } + } + } + + pub async fn bitswap_fetch_hash(&mut self, hash: H256) -> Vec { + let bitswap::Response::Hash(data) = + self.bitswap_request(bitswap::Request::Hash(hash)).await; + data + } + + pub async fn fetch_compact_mb( + &mut self, + mb_hash: H256, + ) -> Option { + match self + .mb_sync_request(mb_sync::Request::CompactMb(mb_hash)) + .await + { + mb_sync::Response::CompactMb(compact) => compact, + _ => unreachable!("CompactMb request must return CompactMb response"), + } + } + + pub async fn fetch_mb_program_states(&mut self, mb_hash: H256) -> Option { + match self + .mb_sync_request(mb_sync::Request::ProgramStates(mb_hash)) + .await + { + mb_sync::Response::ProgramStates(program_states) => program_states, + _ => unreachable!("MB program states request must return MB program states response"), + } + } + + pub async fn fetch_mb_outcome(&mut self, mb_hash: H256) -> Option> { + match self + .mb_sync_request(mb_sync::Request::Outcome(mb_hash)) + .await + { + mb_sync::Response::Outcome(outcome) => outcome, + _ => unreachable!("MB outcome request must return MB outcome response"), + } + } + + pub async fn fetch_mb_schedule(&mut self, mb_hash: H256) -> Option { + match self + .mb_sync_request(mb_sync::Request::Schedule(mb_hash)) + .await + { + mb_sync::Response::Schedule(schedule) => schedule, + _ => unreachable!("MB schedule request must return MB schedule response"), + } + } + + pub async fn fetch_mb_meta(&mut self, mb_hash: H256) -> ethexe_common::db::MbMeta { + match self.mb_sync_request(mb_sync::Request::Meta(mb_hash)).await { + mb_sync::Response::Meta(meta) => meta, + _ => unreachable!("MB meta request must return MB meta response"), + } + } + /// Refresh validator-era state after the chain head changes. /// /// This updates both validator-message verification and validator @@ -670,6 +782,7 @@ struct BehaviourConfig<'a> { router_address: Address, keypair: identity::Keypair, db: Box, + mb_sync_db: Box, transport_type: TransportType, validator_key: Option, general_signer: Signer, @@ -701,6 +814,8 @@ pub(crate) struct Behaviour { pub gossipsub: gossipsub::Behaviour, // data request protocol pub bitswap: bitswap::Behaviour, + // MB row request protocol + pub mb_sync: mb_sync::Behaviour, // injected transaction shenanigans pub injected: injected::Behaviour, // validator discovery @@ -713,6 +828,7 @@ impl Behaviour { router_address, keypair, db, + mb_sync_db, transport_type, validator_key, general_signer, @@ -778,6 +894,8 @@ impl Behaviour { .with_auto_retry(matches!(transport_type, TransportType::Test)); let bitswap = bitswap::Behaviour::new(db, bitswap); + let mb_sync = mb_sync::Behaviour::new(mb_sync_db); + let injected = injected::Behaviour::new(); let validator_discovery = validator::discovery::Config { @@ -800,6 +918,7 @@ impl Behaviour { kad, gossipsub, bitswap, + mb_sync, injected, validator_discovery, }) @@ -828,6 +947,7 @@ mod tests { latest_validators: ValidatorsVec, signer: Signer, validator_key: Option, + bootstrap_addresses: HashSet, } impl NetworkServiceBuilder { @@ -837,9 +957,15 @@ mod tests { latest_validators: nonempty![Address::default()].into(), signer: Signer::memory(), validator_key: None, + bootstrap_addresses: HashSet::new(), } } + fn bootstrap_address(mut self, address: Multiaddr) -> Self { + self.bootstrap_addresses.insert(address); + self + } + fn build(self) -> NetworkService { const GENESIS_BLOCK_HEADER: BlockHeader = BlockHeader { height: 0, @@ -858,6 +984,7 @@ mod tests { latest_validators, signer, validator_key, + bootstrap_addresses, } = self; db.set_config(DBConfig { @@ -866,7 +993,8 @@ mod tests { }); let key = signer.generate().unwrap(); - let config = NetworkConfig::new_test(key, Address::default()); + let mut config = NetworkConfig::new_test(key, Address::default()); + config.bootstrap_addresses = bootstrap_addresses; let runtime_config = NetworkRuntimeConfig { latest_block_header: GENESIS_BLOCK_HEADER, @@ -931,6 +1059,63 @@ mod tests { ); } + #[tokio::test] + async fn request_db_data_from_bootstrap_peer() { + init_logger(); + + let service2 = NetworkServiceBuilder::new(); + let hello = service2.db.cas().write(b"hello"); + let mut service2 = service2.build(); + let service2_peer_id = service2.local_peer_id(); + let (service2_addr, _) = service2.swarm.listen().with_memory_addr_external().await; + let service2_addr = service2_addr.with_p2p(service2_peer_id).unwrap(); + + let service1 = NetworkServiceBuilder::new().bootstrap_address(service2_addr); + let service1 = service1.build(); + let service1_handle = service1.bitswap_handle(); + + tokio::spawn(service1.loop_on_next()); + tokio::spawn(service2.loop_on_next()); + + let response = timeout(Duration::from_secs(5), service1_handle.request(hello)) + .await + .expect("time has elapsed"); + assert_eq!(response, bitswap::Response::Hash(b"hello".to_vec())); + } + + #[tokio::test] + async fn request_compact_mb_from_bootstrap_peer() { + init_logger(); + + let service2 = NetworkServiceBuilder::new(); + let mb_hash = H256::random(); + let compact = CompactMb { + parent: H256::random(), + height: 42, + transactions_hash: H256::random(), + }; + service2.db.set_mb_compact_block(mb_hash, compact); + let mut service2 = service2.build(); + let service2_peer_id = service2.local_peer_id(); + let (service2_addr, _) = service2.swarm.listen().with_memory_addr_external().await; + let service2_addr = service2_addr.with_p2p(service2_peer_id).unwrap(); + + let service1 = NetworkServiceBuilder::new().bootstrap_address(service2_addr); + let service1 = service1.build(); + let service1_handle = service1.mb_sync_handle(); + + tokio::spawn(service1.loop_on_next()); + tokio::spawn(service2.loop_on_next()); + + let response = timeout( + Duration::from_secs(5), + service1_handle.request(mb_sync::Request::CompactMb(mb_hash)), + ) + .await + .expect("time has elapsed"); + assert_eq!(response, mb_sync::Response::CompactMb(Some(compact))); + } + #[tokio::test] async fn peer_blocked_by_score() { init_logger(); diff --git a/ethexe/network/src/mb_sync.rs b/ethexe/network/src/mb_sync.rs new file mode 100644 index 00000000000..46961202ae2 --- /dev/null +++ b/ethexe/network/src/mb_sync.rs @@ -0,0 +1,392 @@ +// Copyright (C) Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +//! Request/response synchronization for MB-indexed database rows. + +use crate::utils::ParityScaleCodec; +use ethexe_common::{ + ProgramStates, Schedule, + db::{CompactMb, MbMeta, MbStorageRO}, + gear::StateTransition, +}; +use ethexe_db::Database; +use gprimitives::H256; +use libp2p::{ + Multiaddr, PeerId, StreamProtocol, + core::{Endpoint, transport::PortUse}, + request_response::{self, Message, ProtocolSupport}, + swarm::{ + ConnectionClosed, ConnectionDenied, ConnectionId, FromSwarm, NetworkBehaviour, THandler, + THandlerInEvent, THandlerOutEvent, ToSwarm, + }, +}; +use parity_scale_codec::{Decode, Encode}; +use std::{ + collections::{HashMap, HashSet}, + task::{Context, Poll}, +}; +use tokio::{ + sync::{mpsc, oneshot}, + task::JoinSet, +}; + +const STREAM_PROTOCOL: StreamProtocol = StreamProtocol::new("/ethexe/mb-sync/1.0.0"); + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Encode, Decode)] +pub enum Request { + CompactMb(H256), + ProgramStates(H256), + Outcome(H256), + Schedule(H256), + Meta(H256), +} + +#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode)] +pub enum Response { + CompactMb(Option), + ProgramStates(Option), + Outcome(Option>), + Schedule(Option), + Meta(MbMeta), +} + +impl Response { + fn is_missing(&self) -> bool { + match self { + Self::CompactMb(value) => value.is_none(), + Self::ProgramStates(value) => value.is_none(), + Self::Outcome(value) => value.is_none(), + Self::Schedule(value) => value.is_none(), + Self::Meta(_) => false, + } + } +} + +#[derive(Clone)] +pub struct Handle(mpsc::UnboundedSender<(Request, oneshot::Sender)>); + +impl Handle { + pub async fn request(&self, request: Request) -> Response { + let (tx, rx) = oneshot::channel(); + self.0.send((request, tx)).expect("channel should be open"); + rx.await.expect("channel should be open") + } +} + +#[auto_impl::auto_impl(&, Box)] +pub trait MbSyncDatabase: Send + Sync + MbStorageRO { + fn clone_boxed(&self) -> Box; +} + +impl MbSyncDatabase for Database { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } +} + +type InnerBehaviour = request_response::Behaviour>; + +struct PendingRequest { + request: Request, + channel: oneshot::Sender, + active: Option, + tried_peers: HashSet, +} + +struct OngoingResponse { + peer: PeerId, + channel: request_response::ResponseChannel, + response: Response, +} + +pub struct Behaviour { + inner: InnerBehaviour, + db: Box, + handle: Handle, + rx: mpsc::UnboundedReceiver<(Request, oneshot::Sender)>, + peers: HashSet, + pending: HashMap, + active: HashMap, + next_request_id: u64, + responses: JoinSet, +} + +impl Behaviour { + pub fn new(db: Box) -> Self { + let (tx, rx) = mpsc::unbounded_channel(); + + Self { + inner: InnerBehaviour::new( + [(STREAM_PROTOCOL, ProtocolSupport::Full)], + request_response::Config::default(), + ), + db, + handle: Handle(tx), + rx, + peers: HashSet::new(), + pending: HashMap::new(), + active: HashMap::new(), + next_request_id: 0, + responses: JoinSet::new(), + } + } + + pub fn handle(&self) -> Handle { + self.handle.clone() + } + + fn response_from_db(db: Box, request: Request) -> Response { + match request { + Request::CompactMb(mb_hash) => Response::CompactMb(db.mb_compact_block(mb_hash)), + Request::ProgramStates(mb_hash) => { + Response::ProgramStates(db.mb_program_states(mb_hash)) + } + Request::Outcome(mb_hash) => Response::Outcome(db.mb_outcome(mb_hash)), + Request::Schedule(mb_hash) => Response::Schedule(db.mb_schedule(mb_hash)), + Request::Meta(mb_hash) => Response::Meta(db.mb_meta(mb_hash)), + } + } + + fn next_peer(peers: &HashSet, tried_peers: &mut HashSet) -> Option { + if peers.is_empty() { + return None; + } + + if peers.len() == tried_peers.len() { + tried_peers.clear(); + } + + let peer = peers.difference(tried_peers).next().copied()?; + tried_peers.insert(peer); + Some(peer) + } + + fn poll_pending_requests(&mut self) { + while let Ok((request, channel)) = self.rx.try_recv() { + let request_id = self.next_request_id; + self.next_request_id += 1; + + self.pending.insert( + request_id, + PendingRequest { + request, + channel, + active: None, + tried_peers: HashSet::new(), + }, + ); + } + + for request_id in self.pending.keys().copied().collect::>() { + let Some(pending) = self.pending.get_mut(&request_id) else { + continue; + }; + + if pending.active.is_some() { + continue; + } + + let Some(peer) = Self::next_peer(&self.peers, &mut pending.tried_peers) else { + continue; + }; + + let outbound_request_id = self.inner.send_request(&peer, pending.request); + pending.active = Some(outbound_request_id); + self.active.insert(outbound_request_id, request_id); + } + } + + fn handle_peer_response( + &mut self, + request_id: request_response::OutboundRequestId, + response: Response, + ) { + let Some(local_request_id) = self.active.remove(&request_id) else { + return; + }; + + if response.is_missing() { + if let Some(pending) = self.pending.get_mut(&local_request_id) { + log::trace!( + "MB sync peer returned no data for {:?}, trying another peer", + pending.request + ); + pending.active = None; + } + } else if let Some(pending) = self.pending.remove(&local_request_id) { + let _ = pending.channel.send(response); + } + } + + fn handle_peer_failure(&mut self, request_id: request_response::OutboundRequestId) { + let Some(local_request_id) = self.active.remove(&request_id) else { + return; + }; + + if let Some(pending) = self.pending.get_mut(&local_request_id) { + pending.active = None; + } + } +} + +impl NetworkBehaviour for Behaviour { + type ConnectionHandler = THandler; + type ToSwarm = (); + + fn handle_pending_inbound_connection( + &mut self, + connection_id: ConnectionId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ConnectionDenied> { + self.inner + .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) + } + + fn handle_established_inbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result, ConnectionDenied> { + self.peers.insert(peer); + self.inner.handle_established_inbound_connection( + connection_id, + peer, + local_addr, + remote_addr, + ) + } + + fn handle_pending_outbound_connection( + &mut self, + connection_id: ConnectionId, + maybe_peer: Option, + addresses: &[Multiaddr], + effective_role: Endpoint, + ) -> Result, ConnectionDenied> { + self.inner.handle_pending_outbound_connection( + connection_id, + maybe_peer, + addresses, + effective_role, + ) + } + + fn handle_established_outbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + addr: &Multiaddr, + role_override: Endpoint, + port_use: PortUse, + ) -> Result, ConnectionDenied> { + self.peers.insert(peer); + self.inner.handle_established_outbound_connection( + connection_id, + peer, + addr, + role_override, + port_use, + ) + } + + fn on_swarm_event(&mut self, event: FromSwarm) { + if let FromSwarm::ConnectionClosed(ConnectionClosed { + peer_id, + remaining_established, + .. + }) = &event + && *remaining_established == 0 + { + self.peers.remove(peer_id); + } + + self.inner.on_swarm_event(event); + } + + fn on_connection_handler_event( + &mut self, + peer_id: PeerId, + connection_id: ConnectionId, + event: THandlerOutEvent, + ) { + self.inner + .on_connection_handler_event(peer_id, connection_id, event); + } + + fn poll( + &mut self, + cx: &mut Context<'_>, + ) -> Poll>> { + self.poll_pending_requests(); + + if let Poll::Ready(Some(response)) = self.responses.poll_join_next(cx) { + let OngoingResponse { + peer, + channel, + response, + } = response.expect("database read panicked"); + let _ = self.inner.send_response(channel, response); + log::trace!("sent MB sync response to {peer}"); + } + + if let Poll::Ready(to_swarm) = self.inner.poll(cx) { + return match to_swarm { + ToSwarm::GenerateEvent(event) => { + match event { + request_response::Event::Message { + peer, + connection_id: _, + message: + Message::Request { + request_id: _, + request, + channel, + }, + } => { + let db = self.db.clone_boxed(); + self.responses.spawn_blocking(move || OngoingResponse { + peer, + channel, + response: Self::response_from_db(db, request), + }); + } + request_response::Event::Message { + peer: _, + connection_id: _, + message: + Message::Response { + request_id, + response, + }, + } => { + self.handle_peer_response(request_id, response); + } + request_response::Event::OutboundFailure { + peer: _, + connection_id: _, + request_id, + error, + } => { + log::trace!("MB sync outbound request {request_id:?} failed: {error}"); + self.handle_peer_failure(request_id); + } + request_response::Event::InboundFailure { .. } + | request_response::Event::ResponseSent { .. } => {} + } + + Poll::Pending + } + to_swarm => { + Poll::Ready(to_swarm.map_out(|_event| { + unreachable!("`ToSwarm::GenerateEvent` is handled above") + })) + } + }; + } + + Poll::Pending + } +} diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 622a890f0cb..e5d1135718b 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -1,16 +1,784 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -//! Fast synchronization stub — always fails so `--fast-sync` doesn't -//! silently degrade to a full-from-genesis catch-up. +//! Fast synchronization for MB-driven ethexe nodes. use crate::Service; -use anyhow::{Result, bail}; +use alloy::eips::BlockId; +use anyhow::{Context, Result, bail, ensure}; +use ethexe_common::{ + BlockData, CodeAndIdUnchecked, Digest, ProgramStates, SimpleBlockData, + db::{ + BlockMetaStorageRO, CodesStorageRO, CodesStorageRW, CompactMb, ConfigStorageRO, + GlobalsStorageRW, MbMeta, MbStorageRO, MbStorageRW, OnChainStorageRW, PreparedBlockData, + }, + events::{ + BlockEvent, RouterEvent, + router::{BatchCommittedEvent, EBCommittedEvent, MBCommittedEvent}, + }, + injected, +}; +use ethexe_compute::ComputeService; +use ethexe_db::{ + Database, + iterator::{ + DatabaseIteratorError, DatabaseIteratorStorage, DispatchStashNode, MailboxNode, + MbOutcomeNode, MbProgramStatesNode, MbScheduleNode, MemoryPagesNode, MemoryPagesRegionNode, + MessageQueueNode, ProgramStateNode, UserMailboxNode, WaitlistNode, + }, + visitor::DatabaseVisitor, +}; +use ethexe_ethereum::router::RouterQuery; +use ethexe_network::NetworkService; +use ethexe_observer::{ObserverService, utils::BlockLoader}; +use ethexe_runtime_common::state::{ + DispatchStash, Mailbox, MemoryPages, MemoryPagesRegion, MessageQueue, ProgramState, + UserMailbox, Waitlist, +}; +use futures::StreamExt; +use gprimitives::{ActorId, CodeId, H256}; +use parity_scale_codec::Decode; +use std::{ + cmp::max, + collections::{BTreeMap, BTreeSet, HashMap}, +}; +use tokio::time::{Duration, timeout}; -// TODO: #5487 implement the actual fast-sync logic. -pub(crate) async fn sync(_service: &mut Service) -> Result<()> { - bail!( - "fast-sync is not implemented for the MB-driven recovery path yet; \ - start the node without --fast-sync (or omit `fast_sync = true` in config)" +struct EventData { + latest_committed_batch: Digest, + committed_mbs: Vec, + latest_committed_eb: Option, +} + +impl EventData { + async fn collect( + block_loader: &impl BlockLoader, + db: &Database, + highest_block: H256, + ) -> Result> { + let mut latest_committed_batch = None; + let mut committed_mbs = Vec::new(); + let mut latest_committed_eb = None; + + let mut block = highest_block; + while !db.block_meta(block).prepared { + let block_data = block_loader.load(block, None).await?; + + for event in block_data.events.iter().rev() { + match event { + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest, + })) if latest_committed_batch.is_none() => { + latest_committed_batch = Some(*digest); + } + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb_hash))) => { + committed_mbs.push(*mb_hash); + } + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(eb_hash))) + if latest_committed_eb.is_none() => + { + latest_committed_eb = Some(*eb_hash); + } + _ => {} + } + } + + block = block_data.header.parent_hash; + if block.is_zero() { + break; + } + } + + if committed_mbs.is_empty() { + return Ok(None); + } + + Ok(Some(Self { + latest_committed_batch: latest_committed_batch.unwrap_or_default(), + committed_mbs, + latest_committed_eb, + })) + } +} + +async fn collect_program_code_ids( + observer: &mut ObserverService, + genesis_block: u32, + latest_committed_block: u32, +) -> Result> { + Ok(observer + .router_query() + .events() + .program_created() + .from_block(genesis_block) + .to_block(latest_committed_block) + .query() + .await + .context("failed to query ProgramCreated events")? + .into_iter() + .map(|(event, _log)| (event.actor_id, event.code_id)) + .collect()) +} + +async fn collect_code_ids( + observer: &mut ObserverService, + genesis_block: u32, + latest_committed_block: u32, +) -> Result> { + Ok(observer + .router_query() + .events() + .code_got_validated() + .valid(true) + .from_block(genesis_block) + .to_block(latest_committed_block) + .query() + .await + .context("failed to query CodeGotValidated events")? + .into_iter() + .map(|(event, _log)| event.code_id) + .collect()) +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RequestMetadata { + ProgramState, + MemoryPages, + MemoryPagesRegion, + MessageQueue, + Waitlist, + Mailbox, + UserMailbox, + DispatchStash, + Data, +} + +impl RequestMetadata { + fn is_data(self) -> bool { + matches!(self, Self::Data) + } +} + +#[derive(Debug)] +struct RequestManager { + db: Database, + total_completed_requests: u64, + total_pending_requests: u64, + pending_requests: HashMap, + responses: Vec<(RequestMetadata, Vec)>, +} + +impl RequestManager { + fn new(db: Database) -> Self { + Self { + db, + total_completed_requests: 0, + total_pending_requests: 0, + pending_requests: HashMap::new(), + responses: Vec::new(), + } + } + + fn add(&mut self, hash: H256, metadata: RequestMetadata) { + debug_assert_ne!(hash, H256::zero(), "zero hash cannot be requested"); + + let old_metadata = self.pending_requests.insert(hash, metadata); + + if let Some(old_metadata) = old_metadata { + debug_assert_eq!(metadata, old_metadata); + } else { + self.total_pending_requests += 1; + } + } + + async fn request( + &mut self, + network: &mut NetworkService, + ) -> Result)>>> { + let pending_network_requests = self.handle_pending_requests(); + + if !pending_network_requests.is_empty() { + let mut response = Vec::with_capacity(pending_network_requests.len()); + for &hash in pending_network_requests.keys() { + response.push((hash, network.bitswap_fetch_hash(hash).await)); + } + + self.handle_response(pending_network_requests, response)?; + } + + let continue_processing = !(self.pending_requests.is_empty() && self.responses.is_empty()); + if continue_processing { + let responses = self.responses.drain(..).collect::>(); + self.total_completed_requests += responses.len() as u64; + Ok(Some(responses)) + } else { + Ok(None) + } + } + + fn handle_pending_requests(&mut self) -> HashMap { + let mut pending_requests = HashMap::new(); + for (hash, metadata) in self.pending_requests.drain() { + if metadata.is_data() && self.db.cas().contains(hash) { + self.total_completed_requests += 1; + continue; + } + + if let Some(data) = self.db.cas().read(hash) { + self.responses.push((metadata, data)); + continue; + } + + pending_requests.insert(hash, metadata); + } + + pending_requests + } + + fn handle_response( + &mut self, + mut pending_network_requests: HashMap, + data: Vec<(H256, Vec)>, + ) -> Result<()> { + for (hash, data) in data { + let metadata = pending_network_requests + .remove(&hash) + .expect("unknown pending request"); + + let db_hash = self.db.cas().write(&data); + ensure!( + hash == db_hash, + "bitswap returned data with unexpected hash" + ); + + self.responses.push((metadata, data)); + } + + debug_assert!( + pending_network_requests.is_empty(), + "network service must gather all requested hashes" + ); + Ok(()) + } + + fn stats(&self) -> (u64, u64) { + let completed = self.total_completed_requests; + let pending = self.total_pending_requests; + debug_assert!(completed <= pending, "{completed} <= {pending}"); + (completed, pending) + } +} + +impl DatabaseVisitor for RequestManager { + fn db(&self) -> &dyn DatabaseIteratorStorage { + &self.db + } + + fn clone_boxed_db(&self) -> Box { + Box::new(self.db.clone()) + } + + fn on_db_error(&mut self, error: DatabaseIteratorError) { + let (hash, metadata) = match error { + DatabaseIteratorError::NoMemoryPages(hash) => { + (hash.inner(), RequestMetadata::MemoryPages) + } + DatabaseIteratorError::NoMemoryPagesRegion(hash) => { + (hash.inner(), RequestMetadata::MemoryPagesRegion) + } + DatabaseIteratorError::NoPageData(hash) => (hash.inner(), RequestMetadata::Data), + DatabaseIteratorError::NoMessageQueue(hash) => { + (hash.inner(), RequestMetadata::MessageQueue) + } + DatabaseIteratorError::NoWaitlist(hash) => (hash.inner(), RequestMetadata::Waitlist), + DatabaseIteratorError::NoDispatchStash(hash) => { + (hash.inner(), RequestMetadata::DispatchStash) + } + DatabaseIteratorError::NoMailbox(hash) => (hash.inner(), RequestMetadata::Mailbox), + DatabaseIteratorError::NoUserMailbox(hash) => { + (hash.inner(), RequestMetadata::UserMailbox) + } + DatabaseIteratorError::NoAllocations(hash) => (hash.inner(), RequestMetadata::Data), + DatabaseIteratorError::NoProgramState(hash) => (hash, RequestMetadata::ProgramState), + DatabaseIteratorError::NoPayload(hash) => (hash.inner(), RequestMetadata::Data), + DatabaseIteratorError::NoOriginalCode(code_id) => { + (code_id.into(), RequestMetadata::Data) + } + DatabaseIteratorError::NoCodeValid(_) + | DatabaseIteratorError::NoInstrumentedCode(_) + | DatabaseIteratorError::NoCodeMetadata(_) => return, + DatabaseIteratorError::NoBlockHeader(_) + | DatabaseIteratorError::NoBlockEvents(_) + | DatabaseIteratorError::NoBlockCodesQueue(_) + | DatabaseIteratorError::NoMb(_) + | DatabaseIteratorError::NoMbProgramStates(_) + | DatabaseIteratorError::NoMbSchedule(_) + | DatabaseIteratorError::NoMbOutcome(_) + | DatabaseIteratorError::NoProgramCodeId(_) => { + unreachable!("{error:?}") + } + }; + self.add(hash, metadata); + } +} + +impl Drop for RequestManager { + fn drop(&mut self) { + #[cfg(debug_assertions)] + { + assert_eq!(self.total_completed_requests, self.total_pending_requests); + assert!(self.pending_requests.is_empty()); + assert!(self.responses.is_empty()); + } + } +} + +async fn sync_hash(network: &mut NetworkService, db: &Database, hash: H256) -> Result<()> { + if db.cas().contains(hash) { + return Ok(()); + } + + let data = network.bitswap_fetch_hash(hash).await; + let db_hash = db.cas().write(&data); + ensure!( + hash == db_hash, + "bitswap returned data with unexpected hash" ); + Ok(()) +} + +async fn sync_compact_mb( + network: &mut NetworkService, + db: &Database, + mb_hash: H256, +) -> Result { + if let Some(compact) = db.mb_compact_block(mb_hash) { + return Ok(compact); + } + + let compact = network + .fetch_compact_mb(mb_hash) + .await + .with_context(|| format!("compact MB {mb_hash} is unavailable from network peers"))?; + db.set_mb_compact_block(mb_hash, compact); + sync_hash(network, db, compact.transactions_hash).await?; + Ok(compact) +} + +async fn find_latest_computed_mb( + network: &mut NetworkService, + db: &Database, + mut mb_hash: H256, +) -> Result<(H256, MbMeta)> { + loop { + let compact = sync_compact_mb(network, db, mb_hash).await?; + let meta = network.fetch_mb_meta(mb_hash).await; + db.mutate_mb_meta(mb_hash, |stored| *stored = meta.clone()); + + if meta.computed { + return Ok((mb_hash, meta)); + } + + log::warn!( + "Committed MB {mb_hash} is not computed on the serving peer, trying its parent {}", + compact.parent + ); + ensure!( + !compact.parent.is_zero(), + "no computed committed MB is available from the serving peer" + ); + mb_hash = compact.parent; + } +} + +async fn find_syncable_committed_mb( + network: &mut NetworkService, + db: &Database, + committed_mbs: Vec, +) -> Result> { + let mut last_error = None; + for mb_hash in committed_mbs { + match timeout( + Duration::from_secs(6), + find_latest_computed_mb(network, db, mb_hash), + ) + .await + { + Ok(Ok(found)) => return Ok(Some(found)), + Ok(Err(error)) => { + log::warn!("Committed MB {mb_hash} is not syncable: {error:?}"); + last_error = Some(error); + } + Err(_) => { + log::warn!("Timed out while checking whether committed MB {mb_hash} is syncable"); + } + } + } + + if let Some(error) = last_error { + return Err(error).context("failed to find a syncable committed MB"); + } + + Ok(None) +} + +async fn sync_latest_mb( + network: &mut NetworkService, + db: &Database, + mb_hash: H256, + code_ids: &BTreeSet, +) -> Result<(MbMeta, ProgramStates)> { + let compact = sync_compact_mb(network, db, mb_hash).await?; + sync_hash(network, db, compact.transactions_hash).await?; + + let meta = network.fetch_mb_meta(mb_hash).await; + ensure!( + meta.computed, + "latest committed MB {mb_hash} is not computed on the serving peer" + ); + db.mutate_mb_meta(mb_hash, |stored| *stored = meta.clone()); + + let program_states = match db.mb_program_states(mb_hash) { + Some(program_states) => program_states, + None => { + let program_states = network + .fetch_mb_program_states(mb_hash) + .await + .with_context(|| { + format!("program states for MB {mb_hash} are unavailable from network peers") + })?; + db.set_mb_program_states(mb_hash, program_states.clone()); + program_states + } + }; + + let outcome = match db.mb_outcome(mb_hash) { + Some(outcome) => outcome, + None => { + let outcome = network.fetch_mb_outcome(mb_hash).await.with_context(|| { + format!("outcome for MB {mb_hash} is unavailable from network peers") + })?; + db.set_mb_outcome(mb_hash, outcome.clone()); + outcome + } + }; + + let schedule = match db.mb_schedule(mb_hash) { + Some(schedule) => schedule, + None => { + let schedule = network.fetch_mb_schedule(mb_hash).await.with_context(|| { + format!("schedule for MB {mb_hash} is unavailable from network peers") + })?; + db.set_mb_schedule(mb_hash, schedule.clone()); + schedule + } + }; + + let mut manager = RequestManager::new(db.clone()); + for &code_id in code_ids { + manager.add(code_id.into(), RequestMetadata::Data); + } + + ethexe_db::visitor::walk( + &mut manager, + MbProgramStatesNode { + mb_hash, + mb_program_states: program_states.clone(), + }, + ); + ethexe_db::visitor::walk( + &mut manager, + MbOutcomeNode { + mb_hash, + mb_outcome: outcome, + }, + ); + ethexe_db::visitor::walk( + &mut manager, + MbScheduleNode { + mb_hash, + mb_schedule: schedule, + }, + ); + + while let Some(responses) = manager.request(network).await? { + let (completed, pending) = manager.stats(); + log::info!("[{completed:>05} / {pending:>05}] Getting network data"); + + for (metadata, data) in responses { + match metadata { + RequestMetadata::ProgramState => { + let state: ProgramState = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk( + &mut manager, + ProgramStateNode { + program_state: state, + }, + ); + } + RequestMetadata::MemoryPages => { + let memory_pages: MemoryPages = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, MemoryPagesNode { memory_pages }); + } + RequestMetadata::MemoryPagesRegion => { + let memory_pages_region: MemoryPagesRegion = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk( + &mut manager, + MemoryPagesRegionNode { + memory_pages_region, + }, + ); + } + RequestMetadata::MessageQueue => { + let message_queue: MessageQueue = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, MessageQueueNode { message_queue }); + } + RequestMetadata::Waitlist => { + let waitlist: Waitlist = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, WaitlistNode { waitlist }); + } + RequestMetadata::Mailbox => { + let mailbox: Mailbox = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, MailboxNode { mailbox }); + } + RequestMetadata::UserMailbox => { + let user_mailbox: UserMailbox = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, UserMailboxNode { user_mailbox }); + } + RequestMetadata::DispatchStash => { + let dispatch_stash: DispatchStash = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, DispatchStashNode { dispatch_stash }); + } + RequestMetadata::Data => {} + } + } + } + + log::info!("Network data getting is done"); + Ok((meta, program_states)) +} + +async fn instrument_codes( + compute: &mut ComputeService, + db: &Database, + mut code_ids: BTreeSet, +) -> Result<()> { + if code_ids.is_empty() { + log::info!("No codes to instrument. Skipping..."); + return Ok(()); + } + + log::info!("Instrument {} codes", code_ids.len()); + + for &code_id in &code_ids { + let original_code = db + .original_code(code_id) + .with_context(|| format!("code {code_id:?} was not fetched from the network"))?; + compute.process_code(CodeAndIdUnchecked { + code_id, + code: original_code, + }); + } + + while let Some(event) = compute.next().await { + let id = event?.unwrap_code_processed(); + code_ids.remove(&id); + if code_ids.is_empty() { + break; + } + } + + log::info!("Codes instrumentation done"); + Ok(()) +} + +async fn set_tx_pool_data_requirement( + db: &Database, + block_loader: &impl BlockLoader, + latest_committed_block_height: u32, +) -> Result<()> { + let to = latest_committed_block_height as u64; + let from = to.saturating_sub(injected::VALIDITY_WINDOW as u64); + + let blocks = block_loader.load_many(from..=to).await?; + for BlockData { + hash, + header, + events, + } in blocks.into_values() + { + db.set_block_header(hash, header); + db.set_block_events(hash, &events); + } + + Ok(()) +} + +async fn latest_era_with_committed_validators( + db: &Database, + router: &RouterQuery, + block_hash: H256, +) -> Result { + let storage_view = router.storage_view_at(block_hash).await?; + + db.config() + .timelines + .era_from_ts(max( + storage_view + .validationSettings + .validators0 + .useFromTimestamp + .to::(), + storage_view + .validationSettings + .validators1 + .useFromTimestamp + .to::(), + )) + .context("failed to calculate era from validators timestamp") +} + +pub(crate) async fn sync(service: &mut Service) -> Result<()> { + let Service { + observer, + compute, + network, + db, + #[cfg(test)] + sender, + .. + } = service; + let Some(network) = network else { + log::warn!("Network service is disabled. Skipping fast synchronization..."); + return Ok(()); + }; + + log::info!("Fast synchronization is in progress..."); + + let finalized_block = observer + .block_loader() + .load_simple(BlockId::finalized()) + .await + .context("failed to get latest finalized block")? + .hash; + + let genesis_block_hash = db.config().genesis_block_hash; + let genesis_block = observer + .block_loader() + .load_simple(genesis_block_hash) + .await + .context("failed to get genesis block")?; + + let block_loader = observer.block_loader(); + let latest_block = observer + .block_loader() + .load_simple(BlockId::latest()) + .await + .context("failed to get latest block")? + .hash; + + let Some(EventData { + latest_committed_batch, + committed_mbs, + latest_committed_eb, + }) = EventData::collect(&block_loader, db, finalized_block) + .await? + .or(if latest_block == finalized_block { + None + } else { + let latest_event_data = EventData::collect(&block_loader, db, latest_block).await?; + if latest_event_data.is_some() { + log::warn!("No finalized committed MB found; trying latest block candidates"); + } + latest_event_data + }) + else { + log::warn!("No committed MB found. Skipping fast synchronization..."); + return Ok(()); + }; + + let mut latest_syncable_mb = find_syncable_committed_mb(network, db, committed_mbs).await?; + if latest_syncable_mb.is_none() + && latest_block != finalized_block + && let Some(EventData { committed_mbs, .. }) = + EventData::collect(&block_loader, db, latest_block).await? + { + log::warn!("No finalized committed MB is syncable; trying latest block candidates"); + latest_syncable_mb = find_syncable_committed_mb(network, db, committed_mbs).await?; + } + let Some((latest_committed_mb, mb_meta)) = latest_syncable_mb else { + bail!("failed to find a syncable committed MB"); + }; + + let latest_committed_eb = (!mb_meta.last_advanced_eb.is_zero()) + .then_some(mb_meta.last_advanced_eb) + .or(latest_committed_eb) + .unwrap_or(genesis_block_hash); + let BlockData { + hash: block_hash, + header, + events, + } = block_loader.load(latest_committed_eb, None).await?; + + let code_ids = collect_code_ids(observer, genesis_block.header.height, header.height).await?; + let program_code_ids = + collect_program_code_ids(observer, genesis_block.header.height, header.height).await?; + + for (program_id, code_id) in program_code_ids { + db.set_program_code_id(program_id, code_id); + } + + sync_latest_mb(network, db, latest_committed_mb, &code_ids).await?; + + instrument_codes(compute, db, code_ids).await?; + + set_tx_pool_data_requirement(db, &block_loader, header.height).await?; + + let latest_era_with_committed_validators = + latest_era_with_committed_validators(db, &observer.router_query(), block_hash).await?; + + ethexe_common::setup_block_in_db( + db, + block_hash, + PreparedBlockData { + header, + events, + latest_era_with_committed_validators, + codes_queue: Default::default(), + last_committed_batch: latest_committed_batch, + last_committed_mb: latest_committed_mb, + last_committed_eb: latest_committed_eb, + }, + ); + + db.globals_mutate(|globals| { + globals.start_block_hash = block_hash; + globals.latest_synced_eb = SimpleBlockData { + hash: block_hash, + header, + }; + globals.latest_prepared_eb_hash = block_hash; + globals.latest_finalized_mb_hash = latest_committed_mb; + globals.latest_computed_mb_hash = latest_committed_mb; + }); + + log::info!( + "Fast synchronization done: synced to {block_hash:?}, height {:?}, MB {latest_committed_mb}", + header.height + ); + + #[cfg(test)] + sender + .send(crate::tests::utils::TestingEvent::FastSyncDone(block_hash)) + .await; + + Ok(()) } diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 3fbcd84c8d4..dbeb597de53 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -33,6 +33,7 @@ use crate::config::{Config, ConfigPublicKey}; use alloy::{ + eips::BlockId, node_bindings::{Anvil, AnvilInstance}, providers::{ProviderBuilder, RootProvider, ext::AnvilApi}, rpc::types::anvil::Metadata, @@ -55,16 +56,13 @@ use ethexe_malachite::{ InjectedTxMempool, MalachiteConfig, MalachiteEvent, MalachiteService, ValidatorEntry, }; use ethexe_network::{NetworkEvent, NetworkRuntimeConfig, NetworkService}; -use ethexe_observer::{ - ObserverConfig, ObserverEvent, ObserverService, - utils::{BlockId, BlockLoader}, -}; +use ethexe_observer::{ObserverConfig, ObserverEvent, ObserverService, utils::BlockLoader}; use ethexe_processor::{ProcessedCodeInfo, Processor, ProcessorConfig, ValidCodeInfo}; use ethexe_prometheus::{PrometheusEvent, PrometheusService}; use ethexe_rpc::{RpcEvent, RpcServer}; use ethexe_service_utils::{OptionFuture as _, OptionStreamNext as _}; use futures::{FutureExt, StreamExt}; -use gprimitives::{CodeId, H256}; +use gprimitives::CodeId; use gsigner::secp256k1::{Address, PrivateKey, PublicKey, Signer}; use std::{ collections::{BTreeMap, HashMap}, @@ -327,7 +325,7 @@ impl Service { let latest_block = observer .block_loader() - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .context("failed to get latest block")?; @@ -424,7 +422,7 @@ impl Service { let latest_block_data = observer .block_loader() - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .context("failed to get latest block")?; diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 19e0c97b0ce..98ab7fdcfab 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -25,6 +25,7 @@ use ethexe_common::{ mock::*, }; use ethexe_consensus::BatchCommitter; +use ethexe_db::verifier::IntegrityVerifier; use ethexe_ethereum::{EthereumBuilder, TryGetReceipt, router::Router}; use ethexe_rpc::InjectedClient; use ethexe_runtime_common::state::Storage; @@ -3075,8 +3076,119 @@ async fn reply_callback() { } #[tokio::test] -#[ignore = "TODO: #5487 port to MB-driven test harness"] -async fn fast_sync() {} +#[ntest::timeout(180_000)] +async fn fast_sync() { + init_logger(); + + let config = TestEnvConfig { + validators: ValidatorsConfig::PreDefined(3), + network: EnvNetworkConfig::Enabled, + ..Default::default() + }; + let mut env = TestEnv::new(config).await.unwrap(); + + test_info!("Starting validators"); + let mut validators = Vec::new(); + for (i, validator) in env.validators.clone().into_iter().enumerate() { + let mut node = env + .new_node(NodeConfig::named(format!("validator-{i}")).validator(validator)) + .await; + node.start_service().await; + validators.push(node); + } + + let observer_events = env.new_observer_events(); + + test_info!("Creating ping program and finalizing it in MB"); + let code_id = env + .upload_code(demo_ping::WASM_BINARY) + .await + .unwrap() + .wait_for() + .await + .unwrap() + .code_id; + let ping_id = env + .create_program(code_id, 500_000_000_000_000) + .await + .unwrap() + .wait_for() + .await + .unwrap() + .program_id; + let reply = env + .send_message(ping_id, b"PING") + .await + .unwrap() + .wait_for() + .await + .unwrap(); + assert_eq!(reply.payload, b"PONG"); + + let alice_mb = tokio::time::timeout(Duration::from_secs(45), async { + observer_events + .filter_map_block_synced() + .find_map(|event| match event { + BlockEvent::Router(ethexe_common::events::RouterEvent::MBCommitted( + ethexe_common::events::router::MBCommittedEvent(mb_hash), + )) => { + let meta = validators[0].db.mb_meta(mb_hash); + (meta.computed + && !meta.last_advanced_eb.is_zero() + && validators[0].db.mb_program_states(mb_hash).is_some() + && validators[0].db.mb_outcome(mb_hash).is_some() + && validators[0].db.mb_schedule(mb_hash).is_some()) + .then_some(mb_hash) + } + _ => None, + }) + .await + }) + .await + .expect("Alice did not commit an MB that advanced an Ethereum block"); + let fast_sync_target = validators[0].db.mb_meta(alice_mb).last_advanced_eb; + assert!(!fast_sync_target.is_zero()); + test_info!("Fast-sync source MB {alice_mb}, target EB {fast_sync_target}"); + + let source_bootstrap_address = validators[0].multiaddr.clone().unwrap(); + + test_info!("Starting Bob with fast sync"); + let mut bob = env + .new_node( + NodeConfig::named("Bob") + .fast_sync() + .network_bootstrap_address(source_bootstrap_address), + ) + .await; + tokio::time::timeout(Duration::from_secs(45), bob.start_service()) + .await + .expect("Bob did not finish fast sync"); + + let fast_synced_block = bob.latest_fast_synced_block.take().unwrap(); + assert_eq!(fast_synced_block, fast_sync_target); + + let bob_globals = bob.db.globals(); + assert_eq!(bob_globals.latest_computed_mb_hash, alice_mb); + assert_eq!(bob_globals.latest_finalized_mb_hash, alice_mb); + assert_eq!(bob_globals.latest_prepared_eb_hash, fast_sync_target); + drop(bob_globals); + + assert_eq!( + bob.db.mb_program_states(alice_mb), + validators[0].db.mb_program_states(alice_mb) + ); + assert_eq!( + bob.db.mb_schedule(alice_mb), + validators[0].db.mb_schedule(alice_mb) + ); + + IntegrityVerifier::new(bob.db.clone()) + .verify_chain(fast_sync_target, fast_synced_block) + .expect("failed to verify Bob database after fast sync"); + + validators.push(bob); + stop_nodes(validators).await; +} #[tokio::test] #[ignore = "TODO: #5488 port to MB-driven test harness"] diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index f12dfd46406..5bd6468fcc9 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -457,6 +457,7 @@ impl TestEnv { validator_config, rpc: service_rpc_config, fast_sync, + network_bootstrap_addresses: custom_network_bootstrap_addresses, } = config; let db = match db { @@ -477,6 +478,11 @@ impl TestEnv { (format!("/memory/{nonce}"), bootstrap_address.clone()) }) .unzip(); + let network_bootstrap_addresses = if custom_network_bootstrap_addresses.is_empty() { + network_bootstrap_address.into_iter().collect() + } else { + custom_network_bootstrap_addresses + }; let network_public_key = network_address.as_ref().map(|_| { self.signer .generate() @@ -515,7 +521,7 @@ impl TestEnv { validator_config, network_public_key, network_address, - network_bootstrap_address, + network_bootstrap_addresses, service_rpc_config, fast_sync, commitment_delay_limit: self.commitment_delay_limit, @@ -913,6 +919,8 @@ pub struct NodeConfig { pub rpc: Option, /// Do P2P database synchronization before the main loop pub fast_sync: bool, + /// Override bootstrap peer address for tests that need a specific data-serving peer. + pub network_bootstrap_addresses: Vec, } impl NodeConfig { @@ -952,6 +960,11 @@ impl NodeConfig { self.fast_sync = true; self } + + pub fn network_bootstrap_address(mut self, address: impl Into) -> Self { + self.network_bootstrap_addresses.push(address.into()); + self + } } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -1019,7 +1032,7 @@ pub struct Node { validator_config: Option, network_public_key: Option, network_address: Option, - network_bootstrap_address: Option, + network_bootstrap_addresses: Vec, service_rpc_config: Option, fast_sync: bool, commitment_delay_limit: std::num::NonZero, @@ -1223,7 +1236,7 @@ impl Node { .expect("failed to create blob loader") .into_box(); - let wait_for_network = self.network_bootstrap_address.is_some(); + let wait_for_network = !self.network_bootstrap_addresses.is_empty(); let network = self.construct_network_service(latest_block, latest_validators); if let Some(addr) = self.network_address.as_ref() { @@ -1269,17 +1282,33 @@ impl Node { self.running_service_handle = Some(handle); self.shutdown_tx = Some(shutdown_tx); + let mut service_started = false; if self.fast_sync { - self.latest_fast_synced_block = Some( - self.events() - .find_map(|event| event.try_unwrap_fast_sync_done().ok()) - .await, - ); + match self + .events() + .find(|event| { + matches!( + event, + TestingEvent::FastSyncDone(_) | TestingEvent::ServiceStarted + ) + }) + .await + { + TestingEvent::FastSyncDone(block) => { + self.latest_fast_synced_block = Some(block); + } + TestingEvent::ServiceStarted => { + service_started = true; + } + _ => unreachable!("filter only allows fast-sync or service-started events"), + } } - self.events() - .find(|e| matches!(e, TestingEvent::ServiceStarted)) - .await; + if !service_started { + self.events() + .find(|e| matches!(e, TestingEvent::ServiceStarted)) + .await; + } // fast sync implies network has connections if wait_for_network && !self.fast_sync { @@ -1360,10 +1389,11 @@ impl Node { let mut config = NetworkConfig::new_test(network_key, self.eth_cfg.router_address); config.listen_addresses = [multiaddr.clone()].into(); config.external_addresses = [multiaddr.clone()].into(); - if let Some(bootstrap_addr) = self.network_bootstrap_address.as_ref() { - let multiaddr = bootstrap_addr.parse().unwrap(); - config.bootstrap_addresses = [multiaddr].into(); - } + config.bootstrap_addresses = self + .network_bootstrap_addresses + .iter() + .map(|addr| addr.parse().unwrap()) + .collect(); let runtime_config = NetworkRuntimeConfig { latest_block_header: latest_block.header, From 0a7c1ec9a83e2ac80bfdbcc1f001d79a95880414 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Tue, 26 May 2026 16:14:10 +0300 Subject: [PATCH 10/72] Remove mb_sync --- ethexe/network/src/lib.rs | 114 +--------- ethexe/network/src/mb_sync.rs | 392 -------------------------------- ethexe/service/src/fast_sync.rs | 339 ++++++++++++--------------- ethexe/service/src/tests/mod.rs | 12 +- 4 files changed, 149 insertions(+), 708 deletions(-) delete mode 100644 ethexe/network/src/mb_sync.rs diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index 4d55275e7f2..2613376012e 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -23,7 +23,6 @@ mod bitswap; mod gossipsub; mod injected; mod kad; -mod mb_sync; mod peer_score; mod slots; mod utils; @@ -42,10 +41,9 @@ use crate::{ }; use anyhow::{Context, anyhow}; use ethexe_common::{ - Address, BlockHeader, ProgramStates, Schedule, ValidatorsVec, + Address, BlockHeader, ValidatorsVec, db::ConfigStorageRO, ecdsa::PublicKey, - gear::StateTransition, injected::{AddressedInjectedTransaction, SignedCompactPromise}, network::{SignedValidatorMessage, VerifiedValidatorMessage}, }; @@ -275,7 +273,6 @@ impl NetworkService { router_address, keypair: keypair.clone(), db: bitswap::BlockstoreDatabase::clone_boxed(&db), - mb_sync_db: mb_sync::MbSyncDatabase::clone_boxed(&db), transport_type, validator_key, general_signer, @@ -443,7 +440,6 @@ impl NetworkService { BehaviourEvent::Kad(event) => self.handle_kad_event(event), BehaviourEvent::Gossipsub(event) => return self.handle_gossipsub_event(event), BehaviourEvent::Bitswap(_event) => {} - BehaviourEvent::MbSync(_event) => {} BehaviourEvent::Injected(event) => return self.handle_injected_event(event), BehaviourEvent::ValidatorDiscovery(event) => { return self.handle_validator_discovery_event(event); @@ -627,11 +623,6 @@ impl NetworkService { self.swarm.behaviour().bitswap.handle() } - /// Handle used by external services to start MB row requests. - pub fn mb_sync_handle(&self) -> mb_sync::Handle { - self.swarm.behaviour().mb_sync.handle() - } - async fn bitswap_request(&mut self, request: bitswap::Request) -> bitswap::Response { let bitswap = self.bitswap_handle(); let fut = bitswap.request(request); @@ -645,75 +636,12 @@ impl NetworkService { } } - async fn mb_sync_request(&mut self, request: mb_sync::Request) -> mb_sync::Response { - let mb_sync = self.mb_sync_handle(); - let fut = mb_sync.request(request); - tokio::pin!(fut); - - loop { - tokio::select! { - _ = self.select_next_some() => {}, - response = &mut fut => break response, - } - } - } - pub async fn bitswap_fetch_hash(&mut self, hash: H256) -> Vec { let bitswap::Response::Hash(data) = self.bitswap_request(bitswap::Request::Hash(hash)).await; data } - pub async fn fetch_compact_mb( - &mut self, - mb_hash: H256, - ) -> Option { - match self - .mb_sync_request(mb_sync::Request::CompactMb(mb_hash)) - .await - { - mb_sync::Response::CompactMb(compact) => compact, - _ => unreachable!("CompactMb request must return CompactMb response"), - } - } - - pub async fn fetch_mb_program_states(&mut self, mb_hash: H256) -> Option { - match self - .mb_sync_request(mb_sync::Request::ProgramStates(mb_hash)) - .await - { - mb_sync::Response::ProgramStates(program_states) => program_states, - _ => unreachable!("MB program states request must return MB program states response"), - } - } - - pub async fn fetch_mb_outcome(&mut self, mb_hash: H256) -> Option> { - match self - .mb_sync_request(mb_sync::Request::Outcome(mb_hash)) - .await - { - mb_sync::Response::Outcome(outcome) => outcome, - _ => unreachable!("MB outcome request must return MB outcome response"), - } - } - - pub async fn fetch_mb_schedule(&mut self, mb_hash: H256) -> Option { - match self - .mb_sync_request(mb_sync::Request::Schedule(mb_hash)) - .await - { - mb_sync::Response::Schedule(schedule) => schedule, - _ => unreachable!("MB schedule request must return MB schedule response"), - } - } - - pub async fn fetch_mb_meta(&mut self, mb_hash: H256) -> ethexe_common::db::MbMeta { - match self.mb_sync_request(mb_sync::Request::Meta(mb_hash)).await { - mb_sync::Response::Meta(meta) => meta, - _ => unreachable!("MB meta request must return MB meta response"), - } - } - /// Refresh validator-era state after the chain head changes. /// /// This updates both validator-message verification and validator @@ -782,7 +710,6 @@ struct BehaviourConfig<'a> { router_address: Address, keypair: identity::Keypair, db: Box, - mb_sync_db: Box, transport_type: TransportType, validator_key: Option, general_signer: Signer, @@ -814,8 +741,6 @@ pub(crate) struct Behaviour { pub gossipsub: gossipsub::Behaviour, // data request protocol pub bitswap: bitswap::Behaviour, - // MB row request protocol - pub mb_sync: mb_sync::Behaviour, // injected transaction shenanigans pub injected: injected::Behaviour, // validator discovery @@ -828,7 +753,6 @@ impl Behaviour { router_address, keypair, db, - mb_sync_db, transport_type, validator_key, general_signer, @@ -894,8 +818,6 @@ impl Behaviour { .with_auto_retry(matches!(transport_type, TransportType::Test)); let bitswap = bitswap::Behaviour::new(db, bitswap); - let mb_sync = mb_sync::Behaviour::new(mb_sync_db); - let injected = injected::Behaviour::new(); let validator_discovery = validator::discovery::Config { @@ -918,7 +840,6 @@ impl Behaviour { kad, gossipsub, bitswap, - mb_sync, injected, validator_discovery, }) @@ -1083,39 +1004,6 @@ mod tests { assert_eq!(response, bitswap::Response::Hash(b"hello".to_vec())); } - #[tokio::test] - async fn request_compact_mb_from_bootstrap_peer() { - init_logger(); - - let service2 = NetworkServiceBuilder::new(); - let mb_hash = H256::random(); - let compact = CompactMb { - parent: H256::random(), - height: 42, - transactions_hash: H256::random(), - }; - service2.db.set_mb_compact_block(mb_hash, compact); - let mut service2 = service2.build(); - let service2_peer_id = service2.local_peer_id(); - let (service2_addr, _) = service2.swarm.listen().with_memory_addr_external().await; - let service2_addr = service2_addr.with_p2p(service2_peer_id).unwrap(); - - let service1 = NetworkServiceBuilder::new().bootstrap_address(service2_addr); - let service1 = service1.build(); - let service1_handle = service1.mb_sync_handle(); - - tokio::spawn(service1.loop_on_next()); - tokio::spawn(service2.loop_on_next()); - - let response = timeout( - Duration::from_secs(5), - service1_handle.request(mb_sync::Request::CompactMb(mb_hash)), - ) - .await - .expect("time has elapsed"); - assert_eq!(response, mb_sync::Response::CompactMb(Some(compact))); - } - #[tokio::test] async fn peer_blocked_by_score() { init_logger(); diff --git a/ethexe/network/src/mb_sync.rs b/ethexe/network/src/mb_sync.rs deleted file mode 100644 index 46961202ae2..00000000000 --- a/ethexe/network/src/mb_sync.rs +++ /dev/null @@ -1,392 +0,0 @@ -// Copyright (C) Gear Technologies Inc. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -//! Request/response synchronization for MB-indexed database rows. - -use crate::utils::ParityScaleCodec; -use ethexe_common::{ - ProgramStates, Schedule, - db::{CompactMb, MbMeta, MbStorageRO}, - gear::StateTransition, -}; -use ethexe_db::Database; -use gprimitives::H256; -use libp2p::{ - Multiaddr, PeerId, StreamProtocol, - core::{Endpoint, transport::PortUse}, - request_response::{self, Message, ProtocolSupport}, - swarm::{ - ConnectionClosed, ConnectionDenied, ConnectionId, FromSwarm, NetworkBehaviour, THandler, - THandlerInEvent, THandlerOutEvent, ToSwarm, - }, -}; -use parity_scale_codec::{Decode, Encode}; -use std::{ - collections::{HashMap, HashSet}, - task::{Context, Poll}, -}; -use tokio::{ - sync::{mpsc, oneshot}, - task::JoinSet, -}; - -const STREAM_PROTOCOL: StreamProtocol = StreamProtocol::new("/ethexe/mb-sync/1.0.0"); - -#[derive(Debug, Copy, Clone, Eq, PartialEq, Encode, Decode)] -pub enum Request { - CompactMb(H256), - ProgramStates(H256), - Outcome(H256), - Schedule(H256), - Meta(H256), -} - -#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode)] -pub enum Response { - CompactMb(Option), - ProgramStates(Option), - Outcome(Option>), - Schedule(Option), - Meta(MbMeta), -} - -impl Response { - fn is_missing(&self) -> bool { - match self { - Self::CompactMb(value) => value.is_none(), - Self::ProgramStates(value) => value.is_none(), - Self::Outcome(value) => value.is_none(), - Self::Schedule(value) => value.is_none(), - Self::Meta(_) => false, - } - } -} - -#[derive(Clone)] -pub struct Handle(mpsc::UnboundedSender<(Request, oneshot::Sender)>); - -impl Handle { - pub async fn request(&self, request: Request) -> Response { - let (tx, rx) = oneshot::channel(); - self.0.send((request, tx)).expect("channel should be open"); - rx.await.expect("channel should be open") - } -} - -#[auto_impl::auto_impl(&, Box)] -pub trait MbSyncDatabase: Send + Sync + MbStorageRO { - fn clone_boxed(&self) -> Box; -} - -impl MbSyncDatabase for Database { - fn clone_boxed(&self) -> Box { - Box::new(self.clone()) - } -} - -type InnerBehaviour = request_response::Behaviour>; - -struct PendingRequest { - request: Request, - channel: oneshot::Sender, - active: Option, - tried_peers: HashSet, -} - -struct OngoingResponse { - peer: PeerId, - channel: request_response::ResponseChannel, - response: Response, -} - -pub struct Behaviour { - inner: InnerBehaviour, - db: Box, - handle: Handle, - rx: mpsc::UnboundedReceiver<(Request, oneshot::Sender)>, - peers: HashSet, - pending: HashMap, - active: HashMap, - next_request_id: u64, - responses: JoinSet, -} - -impl Behaviour { - pub fn new(db: Box) -> Self { - let (tx, rx) = mpsc::unbounded_channel(); - - Self { - inner: InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ), - db, - handle: Handle(tx), - rx, - peers: HashSet::new(), - pending: HashMap::new(), - active: HashMap::new(), - next_request_id: 0, - responses: JoinSet::new(), - } - } - - pub fn handle(&self) -> Handle { - self.handle.clone() - } - - fn response_from_db(db: Box, request: Request) -> Response { - match request { - Request::CompactMb(mb_hash) => Response::CompactMb(db.mb_compact_block(mb_hash)), - Request::ProgramStates(mb_hash) => { - Response::ProgramStates(db.mb_program_states(mb_hash)) - } - Request::Outcome(mb_hash) => Response::Outcome(db.mb_outcome(mb_hash)), - Request::Schedule(mb_hash) => Response::Schedule(db.mb_schedule(mb_hash)), - Request::Meta(mb_hash) => Response::Meta(db.mb_meta(mb_hash)), - } - } - - fn next_peer(peers: &HashSet, tried_peers: &mut HashSet) -> Option { - if peers.is_empty() { - return None; - } - - if peers.len() == tried_peers.len() { - tried_peers.clear(); - } - - let peer = peers.difference(tried_peers).next().copied()?; - tried_peers.insert(peer); - Some(peer) - } - - fn poll_pending_requests(&mut self) { - while let Ok((request, channel)) = self.rx.try_recv() { - let request_id = self.next_request_id; - self.next_request_id += 1; - - self.pending.insert( - request_id, - PendingRequest { - request, - channel, - active: None, - tried_peers: HashSet::new(), - }, - ); - } - - for request_id in self.pending.keys().copied().collect::>() { - let Some(pending) = self.pending.get_mut(&request_id) else { - continue; - }; - - if pending.active.is_some() { - continue; - } - - let Some(peer) = Self::next_peer(&self.peers, &mut pending.tried_peers) else { - continue; - }; - - let outbound_request_id = self.inner.send_request(&peer, pending.request); - pending.active = Some(outbound_request_id); - self.active.insert(outbound_request_id, request_id); - } - } - - fn handle_peer_response( - &mut self, - request_id: request_response::OutboundRequestId, - response: Response, - ) { - let Some(local_request_id) = self.active.remove(&request_id) else { - return; - }; - - if response.is_missing() { - if let Some(pending) = self.pending.get_mut(&local_request_id) { - log::trace!( - "MB sync peer returned no data for {:?}, trying another peer", - pending.request - ); - pending.active = None; - } - } else if let Some(pending) = self.pending.remove(&local_request_id) { - let _ = pending.channel.send(response); - } - } - - fn handle_peer_failure(&mut self, request_id: request_response::OutboundRequestId) { - let Some(local_request_id) = self.active.remove(&request_id) else { - return; - }; - - if let Some(pending) = self.pending.get_mut(&local_request_id) { - pending.active = None; - } - } -} - -impl NetworkBehaviour for Behaviour { - type ConnectionHandler = THandler; - type ToSwarm = (); - - fn handle_pending_inbound_connection( - &mut self, - connection_id: ConnectionId, - local_addr: &Multiaddr, - remote_addr: &Multiaddr, - ) -> Result<(), ConnectionDenied> { - self.inner - .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) - } - - fn handle_established_inbound_connection( - &mut self, - connection_id: ConnectionId, - peer: PeerId, - local_addr: &Multiaddr, - remote_addr: &Multiaddr, - ) -> Result, ConnectionDenied> { - self.peers.insert(peer); - self.inner.handle_established_inbound_connection( - connection_id, - peer, - local_addr, - remote_addr, - ) - } - - fn handle_pending_outbound_connection( - &mut self, - connection_id: ConnectionId, - maybe_peer: Option, - addresses: &[Multiaddr], - effective_role: Endpoint, - ) -> Result, ConnectionDenied> { - self.inner.handle_pending_outbound_connection( - connection_id, - maybe_peer, - addresses, - effective_role, - ) - } - - fn handle_established_outbound_connection( - &mut self, - connection_id: ConnectionId, - peer: PeerId, - addr: &Multiaddr, - role_override: Endpoint, - port_use: PortUse, - ) -> Result, ConnectionDenied> { - self.peers.insert(peer); - self.inner.handle_established_outbound_connection( - connection_id, - peer, - addr, - role_override, - port_use, - ) - } - - fn on_swarm_event(&mut self, event: FromSwarm) { - if let FromSwarm::ConnectionClosed(ConnectionClosed { - peer_id, - remaining_established, - .. - }) = &event - && *remaining_established == 0 - { - self.peers.remove(peer_id); - } - - self.inner.on_swarm_event(event); - } - - fn on_connection_handler_event( - &mut self, - peer_id: PeerId, - connection_id: ConnectionId, - event: THandlerOutEvent, - ) { - self.inner - .on_connection_handler_event(peer_id, connection_id, event); - } - - fn poll( - &mut self, - cx: &mut Context<'_>, - ) -> Poll>> { - self.poll_pending_requests(); - - if let Poll::Ready(Some(response)) = self.responses.poll_join_next(cx) { - let OngoingResponse { - peer, - channel, - response, - } = response.expect("database read panicked"); - let _ = self.inner.send_response(channel, response); - log::trace!("sent MB sync response to {peer}"); - } - - if let Poll::Ready(to_swarm) = self.inner.poll(cx) { - return match to_swarm { - ToSwarm::GenerateEvent(event) => { - match event { - request_response::Event::Message { - peer, - connection_id: _, - message: - Message::Request { - request_id: _, - request, - channel, - }, - } => { - let db = self.db.clone_boxed(); - self.responses.spawn_blocking(move || OngoingResponse { - peer, - channel, - response: Self::response_from_db(db, request), - }); - } - request_response::Event::Message { - peer: _, - connection_id: _, - message: - Message::Response { - request_id, - response, - }, - } => { - self.handle_peer_response(request_id, response); - } - request_response::Event::OutboundFailure { - peer: _, - connection_id: _, - request_id, - error, - } => { - log::trace!("MB sync outbound request {request_id:?} failed: {error}"); - self.handle_peer_failure(request_id); - } - request_response::Event::InboundFailure { .. } - | request_response::Event::ResponseSent { .. } => {} - } - - Poll::Pending - } - to_swarm => { - Poll::Ready(to_swarm.map_out(|_event| { - unreachable!("`ToSwarm::GenerateEvent` is handled above") - })) - } - }; - } - - Poll::Pending - } -} diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index e5d1135718b..e6cd629711a 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -5,35 +5,40 @@ use crate::Service; use alloy::eips::BlockId; -use anyhow::{Context, Result, bail, ensure}; +use anyhow::{Context, Result, ensure}; use ethexe_common::{ - BlockData, CodeAndIdUnchecked, Digest, ProgramStates, SimpleBlockData, + Address, BlockData, CodeAndIdUnchecked, Digest, ProgramStates, SimpleBlockData, + StateHashWithQueueSize, db::{ BlockMetaStorageRO, CodesStorageRO, CodesStorageRW, CompactMb, ConfigStorageRO, - GlobalsStorageRW, MbMeta, MbStorageRO, MbStorageRW, OnChainStorageRW, PreparedBlockData, + GlobalsStorageRW, MbStorageRO, MbStorageRW, OnChainStorageRW, PreparedBlockData, }, events::{ BlockEvent, RouterEvent, router::{BatchCommittedEvent, EBCommittedEvent, MBCommittedEvent}, }, injected, + malachite::Transactions, }; use ethexe_compute::ComputeService; use ethexe_db::{ Database, iterator::{ DatabaseIteratorError, DatabaseIteratorStorage, DispatchStashNode, MailboxNode, - MbOutcomeNode, MbProgramStatesNode, MbScheduleNode, MemoryPagesNode, MemoryPagesRegionNode, - MessageQueueNode, ProgramStateNode, UserMailboxNode, WaitlistNode, + MemoryPagesNode, MemoryPagesRegionNode, MessageQueueNode, ProgramStateNode, + UserMailboxNode, WaitlistNode, }, visitor::DatabaseVisitor, }; -use ethexe_ethereum::router::RouterQuery; +use ethexe_ethereum::{mirror::MirrorQuery, router::RouterQuery}; use ethexe_network::NetworkService; use ethexe_observer::{ObserverService, utils::BlockLoader}; -use ethexe_runtime_common::state::{ - DispatchStash, Mailbox, MemoryPages, MemoryPagesRegion, MessageQueue, ProgramState, - UserMailbox, Waitlist, +use ethexe_runtime_common::{ + ScheduleRestorer, + state::{ + DispatchStash, Mailbox, MemoryPages, MemoryPagesRegion, MessageQueue, ProgramState, + UserMailbox, Waitlist, + }, }; use futures::StreamExt; use gprimitives::{ActorId, CodeId, H256}; @@ -42,7 +47,6 @@ use std::{ cmp::max, collections::{BTreeMap, BTreeSet, HashMap}, }; -use tokio::time::{Duration, timeout}; struct EventData { latest_committed_batch: Digest, @@ -120,6 +124,33 @@ async fn collect_program_code_ids( .collect()) } +async fn collect_program_state_hashes( + observer: &mut ObserverService, + block_hash: H256, + program_code_ids: &BTreeMap, +) -> Result> { + let mut program_states = BTreeMap::new(); + let provider = observer.provider(); + + for &actor_id in program_code_ids.keys() { + let mirror = Address::try_from(actor_id).expect("invalid actor id"); + let mirror = MirrorQuery::new(provider.clone(), mirror); + + let state_hash = mirror.state_hash_at(block_hash).await.with_context(|| { + format!("failed to get state hash for actor {actor_id} at block {block_hash}") + })?; + + ensure!( + !state_hash.is_zero(), + "state hash is zero for actor {actor_id} at block {block_hash}" + ); + + program_states.insert(actor_id, state_hash); + } + + Ok(program_states) +} + async fn collect_code_ids( observer: &mut ObserverService, genesis_block: u32, @@ -333,93 +364,22 @@ impl Drop for RequestManager { } } -async fn sync_hash(network: &mut NetworkService, db: &Database, hash: H256) -> Result<()> { - if db.cas().contains(hash) { - return Ok(()); - } - - let data = network.bitswap_fetch_hash(hash).await; - let db_hash = db.cas().write(&data); - ensure!( - hash == db_hash, - "bitswap returned data with unexpected hash" - ); - Ok(()) -} - -async fn sync_compact_mb( - network: &mut NetworkService, - db: &Database, - mb_hash: H256, -) -> Result { - if let Some(compact) = db.mb_compact_block(mb_hash) { - return Ok(compact); +fn restore_compact_mb(db: &Database, mb_hash: H256) { + if db.mb_compact_block(mb_hash).is_some() { + return; } - let compact = network - .fetch_compact_mb(mb_hash) - .await - .with_context(|| format!("compact MB {mb_hash} is unavailable from network peers"))?; + let transactions_hash = db.set_transactions(Transactions::default()); + // The committed MB is a fast-sync boundary: chain data gives us the head + // hash, but not the Malachite envelope `(parent, height, payload_hash)`. + // Store a synthetic compact row so DB walks have an anchor without + // pretending to restore the pre-boundary MB chain. + let compact = CompactMb { + parent: H256::zero(), + height: 0, + transactions_hash, + }; db.set_mb_compact_block(mb_hash, compact); - sync_hash(network, db, compact.transactions_hash).await?; - Ok(compact) -} - -async fn find_latest_computed_mb( - network: &mut NetworkService, - db: &Database, - mut mb_hash: H256, -) -> Result<(H256, MbMeta)> { - loop { - let compact = sync_compact_mb(network, db, mb_hash).await?; - let meta = network.fetch_mb_meta(mb_hash).await; - db.mutate_mb_meta(mb_hash, |stored| *stored = meta.clone()); - - if meta.computed { - return Ok((mb_hash, meta)); - } - - log::warn!( - "Committed MB {mb_hash} is not computed on the serving peer, trying its parent {}", - compact.parent - ); - ensure!( - !compact.parent.is_zero(), - "no computed committed MB is available from the serving peer" - ); - mb_hash = compact.parent; - } -} - -async fn find_syncable_committed_mb( - network: &mut NetworkService, - db: &Database, - committed_mbs: Vec, -) -> Result> { - let mut last_error = None; - for mb_hash in committed_mbs { - match timeout( - Duration::from_secs(6), - find_latest_computed_mb(network, db, mb_hash), - ) - .await - { - Ok(Ok(found)) => return Ok(Some(found)), - Ok(Err(error)) => { - log::warn!("Committed MB {mb_hash} is not syncable: {error:?}"); - last_error = Some(error); - } - Err(_) => { - log::warn!("Timed out while checking whether committed MB {mb_hash} is syncable"); - } - } - } - - if let Some(error) = last_error { - return Err(error).context("failed to find a syncable committed MB"); - } - - Ok(None) } async fn sync_latest_mb( @@ -427,80 +387,20 @@ async fn sync_latest_mb( db: &Database, mb_hash: H256, code_ids: &BTreeSet, -) -> Result<(MbMeta, ProgramStates)> { - let compact = sync_compact_mb(network, db, mb_hash).await?; - sync_hash(network, db, compact.transactions_hash).await?; - - let meta = network.fetch_mb_meta(mb_hash).await; - ensure!( - meta.computed, - "latest committed MB {mb_hash} is not computed on the serving peer" - ); - db.mutate_mb_meta(mb_hash, |stored| *stored = meta.clone()); - - let program_states = match db.mb_program_states(mb_hash) { - Some(program_states) => program_states, - None => { - let program_states = network - .fetch_mb_program_states(mb_hash) - .await - .with_context(|| { - format!("program states for MB {mb_hash} are unavailable from network peers") - })?; - db.set_mb_program_states(mb_hash, program_states.clone()); - program_states - } - }; - - let outcome = match db.mb_outcome(mb_hash) { - Some(outcome) => outcome, - None => { - let outcome = network.fetch_mb_outcome(mb_hash).await.with_context(|| { - format!("outcome for MB {mb_hash} is unavailable from network peers") - })?; - db.set_mb_outcome(mb_hash, outcome.clone()); - outcome - } - }; - - let schedule = match db.mb_schedule(mb_hash) { - Some(schedule) => schedule, - None => { - let schedule = network.fetch_mb_schedule(mb_hash).await.with_context(|| { - format!("schedule for MB {mb_hash} is unavailable from network peers") - })?; - db.set_mb_schedule(mb_hash, schedule.clone()); - schedule - } - }; + program_state_hashes: BTreeMap, + block_height: u32, +) -> Result { + restore_compact_mb(db, mb_hash); + let mut restored_cached_queue_sizes = BTreeMap::new(); let mut manager = RequestManager::new(db.clone()); + for &state_hash in program_state_hashes.values() { + manager.add(state_hash, RequestMetadata::ProgramState); + } for &code_id in code_ids { manager.add(code_id.into(), RequestMetadata::Data); } - ethexe_db::visitor::walk( - &mut manager, - MbProgramStatesNode { - mb_hash, - mb_program_states: program_states.clone(), - }, - ); - ethexe_db::visitor::walk( - &mut manager, - MbOutcomeNode { - mb_hash, - mb_outcome: outcome, - }, - ); - ethexe_db::visitor::walk( - &mut manager, - MbScheduleNode { - mb_hash, - mb_schedule: schedule, - }, - ); - while let Some(responses) = manager.request(network).await? { let (completed, pending) = manager.stats(); log::info!("[{completed:>05} / {pending:>05}] Getting network data"); @@ -510,6 +410,14 @@ async fn sync_latest_mb( RequestMetadata::ProgramState => { let state: ProgramState = Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + let state_hash = ethexe_db::hash(&data); + restored_cached_queue_sizes.insert( + state_hash, + ( + state.canonical_queue.cached_queue_size, + state.injected_queue.cached_queue_size, + ), + ); ethexe_db::visitor::walk( &mut manager, ProgramStateNode { @@ -563,7 +471,33 @@ async fn sync_latest_mb( } log::info!("Network data getting is done"); - Ok((meta, program_states)) + + let program_states = program_state_hashes + .into_iter() + .map(|(program_id, hash)| { + let (canonical_queue_size, injected_queue_size) = *restored_cached_queue_sizes + .get(&hash) + .expect("program state cached queue size must be restored"); + ( + program_id, + StateHashWithQueueSize { + hash, + canonical_queue_size, + injected_queue_size, + }, + ) + }) + .collect(); + + let schedule = ScheduleRestorer::from_storage(db, &program_states, block_height)?.restore(); + db.set_mb_program_states(mb_hash, program_states.clone()); + db.set_mb_schedule(mb_hash, schedule); + // The committed fast-sync anchor is never replayed for batch creation, so + // keep queue-derived outcome invariants out of restoration and persist an + // empty row only to satisfy database walks over committed MB metadata. + db.set_mb_outcome(mb_hash, Default::default()); + + Ok(program_states) } async fn instrument_codes( @@ -685,43 +619,38 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { .context("failed to get latest block")? .hash; - let Some(EventData { - latest_committed_batch, - committed_mbs, - latest_committed_eb, - }) = EventData::collect(&block_loader, db, finalized_block) - .await? - .or(if latest_block == finalized_block { - None - } else { - let latest_event_data = EventData::collect(&block_loader, db, latest_block).await?; - if latest_event_data.is_some() { - log::warn!("No finalized committed MB found; trying latest block candidates"); - } + let mut state_query_block = finalized_block; + let event_data = match EventData::collect(&block_loader, db, finalized_block).await? { + Some(event_data) => event_data, + None if latest_block != finalized_block => { + let Some(latest_event_data) = + EventData::collect(&block_loader, db, latest_block).await? + else { + log::warn!("No committed MB found. Skipping fast synchronization..."); + return Ok(()); + }; + log::warn!("No finalized committed MB found; trying latest block candidates"); + state_query_block = latest_block; latest_event_data - }) - else { - log::warn!("No committed MB found. Skipping fast synchronization..."); - return Ok(()); + } + None => { + log::warn!("No committed MB found. Skipping fast synchronization..."); + return Ok(()); + } }; - let mut latest_syncable_mb = find_syncable_committed_mb(network, db, committed_mbs).await?; - if latest_syncable_mb.is_none() - && latest_block != finalized_block - && let Some(EventData { committed_mbs, .. }) = - EventData::collect(&block_loader, db, latest_block).await? - { - log::warn!("No finalized committed MB is syncable; trying latest block candidates"); - latest_syncable_mb = find_syncable_committed_mb(network, db, committed_mbs).await?; - } - let Some((latest_committed_mb, mb_meta)) = latest_syncable_mb else { - bail!("failed to find a syncable committed MB"); - }; + let latest_committed_mb = event_data + .committed_mbs + .first() + .copied() + .expect("EventData always contains at least one committed MB"); + let EventData { + latest_committed_batch, + latest_committed_eb, + .. + } = event_data; - let latest_committed_eb = (!mb_meta.last_advanced_eb.is_zero()) - .then_some(mb_meta.last_advanced_eb) - .or(latest_committed_eb) - .unwrap_or(genesis_block_hash); + let latest_committed_eb = latest_committed_eb.unwrap_or(genesis_block_hash); let BlockData { hash: block_hash, header, @@ -731,12 +660,26 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { let code_ids = collect_code_ids(observer, genesis_block.header.height, header.height).await?; let program_code_ids = collect_program_code_ids(observer, genesis_block.header.height, header.height).await?; + let program_state_hashes = + collect_program_state_hashes(observer, state_query_block, &program_code_ids).await?; - for (program_id, code_id) in program_code_ids { + for (&program_id, &code_id) in &program_code_ids { db.set_program_code_id(program_id, code_id); } - sync_latest_mb(network, db, latest_committed_mb, &code_ids).await?; + sync_latest_mb( + network, + db, + latest_committed_mb, + &code_ids, + program_state_hashes, + header.height, + ) + .await?; + db.mutate_mb_meta(latest_committed_mb, |meta| { + meta.computed = true; + meta.last_advanced_eb = latest_committed_eb; + }); instrument_codes(compute, db, code_ids).await?; diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 98ab7fdcfab..fdb9f3d49db 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -3167,11 +3167,12 @@ async fn fast_sync() { let fast_synced_block = bob.latest_fast_synced_block.take().unwrap(); assert_eq!(fast_synced_block, fast_sync_target); - let bob_globals = bob.db.globals(); - assert_eq!(bob_globals.latest_computed_mb_hash, alice_mb); - assert_eq!(bob_globals.latest_finalized_mb_hash, alice_mb); - assert_eq!(bob_globals.latest_prepared_eb_hash, fast_sync_target); - drop(bob_globals); + { + let bob_globals = bob.db.globals(); + assert_eq!(bob_globals.latest_computed_mb_hash, alice_mb); + assert_eq!(bob_globals.latest_finalized_mb_hash, alice_mb); + assert_eq!(bob_globals.latest_prepared_eb_hash, fast_sync_target); + } assert_eq!( bob.db.mb_program_states(alice_mb), @@ -3181,6 +3182,7 @@ async fn fast_sync() { bob.db.mb_schedule(alice_mb), validators[0].db.mb_schedule(alice_mb) ); + assert_eq!(bob.db.mb_outcome(alice_mb), Some(Default::default())); IntegrityVerifier::new(bob.db.clone()) .verify_chain(fast_sync_target, fast_synced_block) From e9086754e31be4b417a12c6d407709208883262d Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Tue, 26 May 2026 17:28:37 +0300 Subject: [PATCH 11/72] Initial assert_chain --- ethexe/service/src/tests/mod.rs | 94 +++++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 22 deletions(-) diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index fdb9f3d49db..fe463c23748 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -6,7 +6,7 @@ pub(crate) mod utils; use crate::tests::utils::{ - EnvNetworkConfig, InfiniteStreamExt, NodeConfig, TestEnv, TestEnvConfig, TestingEvent, + EnvNetworkConfig, InfiniteStreamExt, Node, NodeConfig, TestEnv, TestEnvConfig, TestingEvent, TestingNetworkEvent, TestingRpcEvent, ValidatorsConfig, init_logger, stop_nodes, test_info, }; use alloy::{ @@ -14,7 +14,10 @@ use alloy::{ providers::{Provider as _, WalletProvider, ext::AnvilApi}, }; use ethexe_common::{ - db::{CodesStorageRO, GlobalsStorageRO, InjectedStorageRO, MbStorageRO, OnChainStorageRO}, + db::{ + BlockMetaStorageRO, CodesStorageRO, GlobalsStorageRO, InjectedStorageRO, MbStorageRO, + OnChainStorageRO, + }, ecdsa::ContractSignature, events::{ BlockEvent, MirrorEvent, @@ -3080,6 +3083,67 @@ async fn reply_callback() { async fn fast_sync() { init_logger(); + let assert_chain = |latest_block, fast_synced_block, latest_mb, alice: &Node, bob: &Node| { + log::info!("Assert chain in range {latest_block}..{fast_synced_block}"); + + IntegrityVerifier::new(alice.db.clone()) + .verify_chain(latest_block, fast_synced_block) + .expect("failed to verify Alice database"); + + IntegrityVerifier::new(bob.db.clone()) + .verify_chain(latest_block, fast_synced_block) + .expect("failed to verify Bob database"); + + let bob_globals = bob.db.globals(); + assert_eq!(bob_globals.latest_computed_mb_hash, latest_mb); + assert_eq!(bob_globals.latest_finalized_mb_hash, latest_mb); + assert_eq!(bob_globals.latest_prepared_eb_hash, latest_block); + + assert_eq!( + bob.db.mb_program_states(latest_mb), + alice.db.mb_program_states(latest_mb) + ); + assert_eq!( + bob.db.mb_schedule(latest_mb), + alice.db.mb_schedule(latest_mb) + ); + assert_eq!(bob.db.mb_outcome(latest_mb), Some(Default::default())); + + let mut block = latest_block; + loop { + if fast_synced_block == block { + break; + } + + log::trace!("assert block {block}"); + + // Check block meta, excluding codes_queue, which can vary. + let alice_meta = alice.db.block_meta(block); + let bob_meta = bob.db.block_meta(block); + assert!( + alice_meta.prepared && bob_meta.prepared, + "Block {block} is not prepared for alice or bob" + ); + assert_eq!( + alice_meta.last_committed_batch, + bob_meta.last_committed_batch + ); + assert_eq!(alice_meta.last_committed_mb, bob_meta.last_committed_mb); + assert_eq!(alice_meta.last_committed_eb, bob_meta.last_committed_eb); + assert_eq!( + alice_meta.latest_era_validators_committed, + bob_meta.latest_era_validators_committed + ); + + assert_eq!(alice.db.block_header(block), bob.db.block_header(block)); + assert_eq!(alice.db.block_events(block), bob.db.block_events(block)); + assert_eq!(alice.db.block_synced(block), bob.db.block_synced(block)); + + let header = alice.db.block_header(block).unwrap(); + block = header.parent_hash; + } + }; + let config = TestEnvConfig { validators: ValidatorsConfig::PreDefined(3), network: EnvNetworkConfig::Enabled, @@ -3166,27 +3230,13 @@ async fn fast_sync() { let fast_synced_block = bob.latest_fast_synced_block.take().unwrap(); assert_eq!(fast_synced_block, fast_sync_target); - - { - let bob_globals = bob.db.globals(); - assert_eq!(bob_globals.latest_computed_mb_hash, alice_mb); - assert_eq!(bob_globals.latest_finalized_mb_hash, alice_mb); - assert_eq!(bob_globals.latest_prepared_eb_hash, fast_sync_target); - } - - assert_eq!( - bob.db.mb_program_states(alice_mb), - validators[0].db.mb_program_states(alice_mb) + assert_chain( + fast_sync_target, + fast_synced_block, + alice_mb, + &validators[0], + &bob, ); - assert_eq!( - bob.db.mb_schedule(alice_mb), - validators[0].db.mb_schedule(alice_mb) - ); - assert_eq!(bob.db.mb_outcome(alice_mb), Some(Default::default())); - - IntegrityVerifier::new(bob.db.clone()) - .verify_chain(fast_sync_target, fast_synced_block) - .expect("failed to verify Bob database after fast sync"); validators.push(bob); stop_nodes(validators).await; From fcac944f23975ae7a705303511fc45c4c3634b4d Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Tue, 2 Jun 2026 15:19:22 +0300 Subject: [PATCH 12/72] make workspace-hack --- Cargo.lock | 141 +++++++++++++++++++++++---- utils/gear-workspace-hack/Cargo.toml | 10 +- 2 files changed, 127 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2131ffffc7e..ebdd917e949 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1224,7 +1224,7 @@ version = "0.7.0-pre" source = "git+https://github.com/circlefin/malachite?rev=1fe7961aca933cefad8e4d9a52f50eda565288e7#1fe7961aca933cefad8e4d9a52f50eda565288e7" dependencies = [ "bs58 0.5.1", - "multihash 0.19.3", + "multihash 0.19.5", "rand 0.8.5", "serde", "thiserror 2.0.17", @@ -2291,6 +2291,32 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +[[package]] +name = "beetswap" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a920844a4696d53e72190a8348f0048a6799647b3448d38bef90b85e0310df0" +dependencies = [ + "asynchronous-codec 0.7.0", + "blockstore", + "bytes", + "cid 0.11.3", + "fnv", + "futures-core", + "futures-timer", + "futures-util", + "libp2p-core 0.43.2", + "libp2p-identity", + "libp2p-swarm 0.47.0", + "multihash-codetable", + "quick-protobuf", + "smallvec", + "thiserror 2.0.17", + "tracing", + "unsigned-varint 0.8.0", + "web-time", +] + [[package]] name = "bimap" version = "0.6.3" @@ -2514,6 +2540,18 @@ dependencies = [ "piper", ] +[[package]] +name = "blockstore" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509096e88e431095763b3f5ee1e1cdb09212e4d5b2eccc91ddea965deefedb7d" +dependencies = [ + "cid 0.11.3", + "dashmap 6.1.0", + "multihash 0.19.5", + "thiserror 2.0.17", +] + [[package]] name = "blst" version = "0.3.16" @@ -3063,6 +3101,17 @@ dependencies = [ "unsigned-varint 0.7.2", ] +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash 0.19.5", + "unsigned-varint 0.8.0", +] + [[package]] name = "cipher" version = "0.2.5" @@ -5755,17 +5804,18 @@ dependencies = [ "assert_matches", "async-trait", "auto_impl", + "beetswap", + "blockstore", + "cid 0.11.3", "derive_more 2.1.1", "ethexe-common", "ethexe-db", - "ethexe-service-utils", "futures", "gear-workspace-hack", "gprimitives", "gsigner", "indexmap 2.14.0", "ip_network", - "itertools 0.13.0", "libp2p 0.56.0", "libp2p-gossipsub", "libp2p-swarm-test", @@ -5773,6 +5823,7 @@ dependencies = [ "lru 0.16.3", "metrics", "metrics-derive", + "multihash 0.19.5", "nonempty 0.12.0", "parity-scale-codec", "prometheus-client 0.23.1", @@ -5829,7 +5880,6 @@ version = "1.10.0" dependencies = [ "alloy", "anyhow", - "derive_more 2.1.1", "ethexe-common", "ethexe-db", "ethexe-ethereum", @@ -7842,6 +7892,7 @@ dependencies = [ "clap_builder", "concurrent-queue", "const-hex", + "core2", "cranelift-bitset", "crc32fast", "crossbeam-channel", @@ -7941,7 +7992,7 @@ dependencies = [ "merlin", "miniz_oxide", "mio", - "multihash 0.19.3", + "multihash 0.19.5", "names", "nix", "nom 7.1.3", @@ -10254,7 +10305,7 @@ dependencies = [ "libp2p-identity", "log", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "multistream-select", "once_cell", "parking_lot 0.12.5", @@ -10280,7 +10331,7 @@ dependencies = [ "futures-timer", "libp2p-identity", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "multistream-select", "parking_lot 0.12.5", "pin-project", @@ -10412,7 +10463,7 @@ dependencies = [ "ed25519-dalek", "hkdf", "k256", - "multihash 0.19.3", + "multihash 0.19.5", "p256", "quick-protobuf", "rand 0.8.5", @@ -10570,7 +10621,7 @@ dependencies = [ "libp2p-identity", "log", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "once_cell", "quick-protobuf", "rand 0.8.5", @@ -10594,7 +10645,7 @@ dependencies = [ "libp2p-core 0.43.2", "libp2p-identity", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "quick-protobuf", "rand 0.8.5", "snow", @@ -11820,7 +11871,7 @@ dependencies = [ "data-encoding", "libp2p-identity", "multibase", - "multihash 0.19.3", + "multihash 0.19.5", "percent-encoding", "serde", "static_assertions", @@ -11851,7 +11902,7 @@ dependencies = [ "blake3", "core2", "digest 0.10.7", - "multihash-derive", + "multihash-derive 0.8.1", "sha2 0.10.9", "sha3", "unsigned-varint 0.7.2", @@ -11868,7 +11919,7 @@ dependencies = [ "blake3", "core2", "digest 0.10.7", - "multihash-derive", + "multihash-derive 0.8.1", "sha2 0.10.9", "sha3", "unsigned-varint 0.7.2", @@ -11876,15 +11927,33 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.3" +version = "0.19.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" dependencies = [ - "core2", "serde", "unsigned-varint 0.8.0", ] +[[package]] +name = "multihash-codetable" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67996849749d25f1da9f238e8ace2ece8f9d6bdf3f9750aaf2ae7de3a5cad8ea" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "blake3", + "core2", + "digest 0.10.7", + "multihash-derive 0.9.3", + "ripemd", + "sha1", + "sha2 0.10.9", + "sha3", + "strobe-rs", +] + [[package]] name = "multihash-derive" version = "0.8.1" @@ -11899,6 +11968,29 @@ dependencies = [ "synstructure 0.12.6", ] +[[package]] +name = "multihash-derive" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4723acdce756db211a53f01b3419dbdf63cb48cef5df86260f55309364735fbf" +dependencies = [ + "multihash 0.19.5", + "multihash-derive-impl", +] + +[[package]] +name = "multihash-derive-impl" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f932556f78452e5604cef711349d337ec081a9aa3c96e67b3127c8f5df05d550" +dependencies = [ + "proc-macro-crate 3.4.0", + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure 0.13.2", +] + [[package]] name = "multimap" version = "0.8.3" @@ -15810,7 +15902,7 @@ dependencies = [ "libp2p 0.52.4", "linked_hash_set", "log", - "multihash 0.19.3", + "multihash 0.19.5", "parity-scale-codec", "prost 0.12.6", "prost-build 0.12.6", @@ -16459,7 +16551,7 @@ dependencies = [ "litep2p", "log", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "rand 0.8.5", "thiserror 1.0.69", "zeroize", @@ -18683,6 +18775,19 @@ dependencies = [ "serde", ] +[[package]] +name = "strobe-rs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98fe17535ea31344936cc58d29fec9b500b0452ddc4cc24c429c8a921a0e84e5" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "keccak", + "subtle 2.6.1", + "zeroize", +] + [[package]] name = "strsim" version = "0.8.0" diff --git a/utils/gear-workspace-hack/Cargo.toml b/utils/gear-workspace-hack/Cargo.toml index 7eee68f341c..fa00ca2a5a8 100644 --- a/utils/gear-workspace-hack/Cargo.toml +++ b/utils/gear-workspace-hack/Cargo.toml @@ -259,9 +259,11 @@ clap = { version = "4", features = ["derive", "env", "string", "wrap_help"] } clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "string", "suggestions", "usage", "wrap_help"] } concurrent-queue = { version = "2" } const-hex = { version = "1", features = ["core-error", "serde"] } +core2 = { git = "https://github.com/bbqsrc/core2", rev = "545e84bcb0f235b12e21351e0c69767958efe2a7", default-features = false, features = ["std"] } cranelift-bitset = { version = "0.131", default-features = false, features = ["enable-serde"] } crc32fast = { version = "1" } crossbeam-channel = { version = "0.5" } +crossbeam-epoch = { version = "0.9" } crossbeam-utils = { version = "0.8" } crunchy = { version = "0.2", features = ["std"] } crypto-common = { version = "0.1", default-features = false, features = ["getrandom", "std"] } @@ -548,9 +550,11 @@ clap = { version = "4", features = ["derive", "env", "string", "wrap_help"] } clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "string", "suggestions", "usage", "wrap_help"] } concurrent-queue = { version = "2" } const-hex = { version = "1", features = ["core-error", "serde"] } +core2 = { git = "https://github.com/bbqsrc/core2", rev = "545e84bcb0f235b12e21351e0c69767958efe2a7", default-features = false, features = ["std"] } cranelift-bitset = { version = "0.131", default-features = false, features = ["enable-serde"] } crc32fast = { version = "1" } crossbeam-channel = { version = "0.5" } +crossbeam-epoch = { version = "0.9" } crossbeam-utils = { version = "0.8" } crunchy = { version = "0.2", features = ["std"] } crypto-common = { version = "0.1", default-features = false, features = ["getrandom", "std"] } @@ -796,7 +800,6 @@ x25519-dalek = { version = "2", features = ["static_secrets"] } zeroize = { version = "1", features = ["derive", "std"] } [target.'cfg(all(target_arch = "x86_64", target_vendor = "unknown", target_os = "linux", target_env = "gnu", not(any(target_arch = "wasm32", loom))))'.dependencies] -crossbeam-epoch = { version = "0.9" } errno = { version = "0.3" } hickory-proto = { version = "0.25", default-features = false, features = ["mdns", "tokio"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "logging", "ring", "tls12", "webpki-tokio"] } @@ -821,7 +824,6 @@ unicode-normalization = { version = "0.1" } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.'cfg(all(target_arch = "x86_64", target_vendor = "unknown", target_os = "linux", target_env = "gnu", not(any(target_arch = "wasm32", loom))))'.build-dependencies] -crossbeam-epoch = { version = "0.9" } errno = { version = "0.3" } hickory-proto = { version = "0.25", default-features = false, features = ["mdns", "tokio"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "logging", "ring", "tls12", "webpki-tokio"] } @@ -846,7 +848,6 @@ unicode-normalization = { version = "0.1" } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.'cfg(all(target_arch = "aarch64", target_vendor = "unknown", target_os = "linux", target_env = "gnu", not(any(target_arch = "wasm32", loom))))'.dependencies] -crossbeam-epoch = { version = "0.9" } errno = { version = "0.3" } hickory-proto = { version = "0.25", default-features = false, features = ["mdns", "tokio"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "logging", "ring", "tls12", "webpki-tokio"] } @@ -871,7 +872,6 @@ unicode-normalization = { version = "0.1" } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.'cfg(all(target_arch = "aarch64", target_vendor = "unknown", target_os = "linux", target_env = "gnu", not(any(target_arch = "wasm32", loom))))'.build-dependencies] -crossbeam-epoch = { version = "0.9" } errno = { version = "0.3" } hickory-proto = { version = "0.25", default-features = false, features = ["mdns", "tokio"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "logging", "ring", "tls12", "webpki-tokio"] } @@ -896,7 +896,6 @@ unicode-normalization = { version = "0.1" } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.'cfg(all(target_arch = "aarch64", target_vendor = "apple", target_os = "macos", not(any(target_arch = "wasm32", loom))))'.dependencies] -crossbeam-epoch = { version = "0.9" } errno = { version = "0.3" } hickory-proto = { version = "0.25", default-features = false, features = ["mdns", "tokio"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "logging", "ring", "tls12", "webpki-tokio"] } @@ -919,7 +918,6 @@ unicode-normalization = { version = "0.1" } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [target.'cfg(all(target_arch = "aarch64", target_vendor = "apple", target_os = "macos", not(any(target_arch = "wasm32", loom))))'.build-dependencies] -crossbeam-epoch = { version = "0.9" } errno = { version = "0.3" } hickory-proto = { version = "0.25", default-features = false, features = ["mdns", "tokio"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "logging", "ring", "tls12", "webpki-tokio"] } From e74c8eb31fa2dfbe95ba7eaecbea3555ecd2dbac Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 4 Jun 2026 14:43:04 +0300 Subject: [PATCH 13/72] Pending removal of `custom_network_bootstrap_addresses` --- ethexe/service/src/lib.rs | 13 +- ethexe/service/src/tests/mod.rs | 203 +++++++++++++++++++------- ethexe/service/src/tests/utils/env.rs | 27 +--- 3 files changed, 169 insertions(+), 74 deletions(-) diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 281bd71770b..73a15911883 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -42,7 +42,7 @@ use anyhow::{Context, Result, bail}; use ethexe_blob_loader::{BlobLoader, BlobLoaderEvent, BlobLoaderService, ConsensusLayerConfig}; use ethexe_common::{ CodeAndIdUnchecked, PromiseEmissionMode, - db::{GlobalsStorageRW, MbStorageRO, OnChainStorageRO}, + db::{GlobalsStorageRO, GlobalsStorageRW, MbStorageRO, OnChainStorageRO}, injected::{CompactPromise, Receipt}, network::VerifiedValidatorMessage, }; @@ -581,6 +581,17 @@ impl Service { pub async fn run(mut self) -> Result<()> { if self.fast_sync { fast_sync::sync(&mut self).await?; + + let block_hash = self.db.globals().latest_prepared_eb_hash; + if !block_hash.is_zero() { + if let Some(malachite) = self.malachite.as_mut() { + let block = self + .db + .block_simple_data(block_hash) + .context("Cannot find fast-synced chain head")?; + malachite.receive_new_chain_head(block); + } + } } self.run_inner().await.inspect_err(|err| { diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 27096dceb41..0e6755b88af 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -7,7 +7,8 @@ pub(crate) mod utils; use crate::tests::utils::{ EnvNetworkConfig, InfiniteStreamExt, Node, NodeConfig, TestEnv, TestEnvConfig, TestingEvent, - TestingNetworkEvent, TestingRpcEvent, ValidatorsConfig, init_logger, stop_nodes, test_info, + TestingEventReceiver, TestingNetworkEvent, TestingRpcEvent, ValidatorsConfig, init_logger, + stop_nodes, test_info, }; use alloy::{ primitives::U256, @@ -3169,66 +3170,155 @@ async fn reply_callback() { async fn fast_sync() { init_logger(); - let assert_chain = |latest_block, fast_synced_block, latest_mb, alice: &Node, bob: &Node| { - log::info!("Assert chain in range {latest_block}..{fast_synced_block}"); + fn block_prepared(db: ðexe_db::Database, block: H256) -> bool { + db.block_header(block).is_some() + && db.block_events(block).is_some() + && db.block_synced(block) + && db.block_meta(block).prepared + } + + async fn wait_till_block_prepared(receiver: &mut TestingEventReceiver, block: H256) { + if block_prepared(receiver.db(), block) { + return; + } + + receiver + .find_map_with_db(|db, _| block_prepared(&db, block).then_some(())) + .await; + } + + let assert_fast_sync_boundary = |fast_synced_block, alice: &Node, bob: &Node| { + log::info!("Assert fast-sync boundary at {fast_synced_block}"); IntegrityVerifier::new(alice.db.clone()) - .verify_chain(latest_block, fast_synced_block) + .verify_chain(fast_synced_block, fast_synced_block) .expect("failed to verify Alice database"); IntegrityVerifier::new(bob.db.clone()) - .verify_chain(latest_block, fast_synced_block) + .verify_chain(fast_synced_block, fast_synced_block) .expect("failed to verify Bob database"); let bob_globals = bob.db.globals(); - assert_eq!(bob_globals.latest_computed_mb_hash, latest_mb); + let latest_mb = bob_globals.latest_computed_mb_hash; + assert_ne!(latest_mb, H256::zero()); assert_eq!(bob_globals.latest_finalized_mb_hash, latest_mb); - assert_eq!(bob_globals.latest_prepared_eb_hash, latest_block); + assert_eq!(bob_globals.latest_prepared_eb_hash, fast_synced_block); + drop(bob_globals); + + let bob_mb_meta = bob.db.mb_meta(latest_mb); + assert!(bob_mb_meta.computed); + assert_eq!(bob_mb_meta.last_advanced_eb, fast_synced_block); + assert!(bob.db.mb_program_states(latest_mb).is_some()); + assert!(bob.db.mb_schedule(latest_mb).is_some()); + assert_eq!(bob.db.mb_outcome(latest_mb), Some(Default::default())); - assert_eq!( - bob.db.mb_program_states(latest_mb), - alice.db.mb_program_states(latest_mb) + if let Some(alice_program_states) = alice.db.mb_program_states(latest_mb) { + assert_eq!( + Some(alice_program_states), + bob.db.mb_program_states(latest_mb) + ); + } + if let Some(alice_schedule) = alice.db.mb_schedule(latest_mb) { + assert_eq!(Some(alice_schedule), bob.db.mb_schedule(latest_mb)); + } + + // Compare the EB/block database surface with Alice while letting + // fast-sync boundary metadata vary, same as the old announce-era test + // allowed codes_queue and announce-specific rows to differ. Bob + // restores these fields at its boundary block instead of replaying + // the later block where Alice observed the commitment. + let alice_meta = alice.db.block_meta(fast_synced_block); + let bob_meta = bob.db.block_meta(fast_synced_block); + assert!( + alice_meta.prepared && bob_meta.prepared, + "Block {fast_synced_block} is not prepared for Alice or Bob" ); assert_eq!( - bob.db.mb_schedule(latest_mb), - alice.db.mb_schedule(latest_mb) + alice_meta.latest_era_validators_committed, + bob_meta.latest_era_validators_committed ); - assert_eq!(bob.db.mb_outcome(latest_mb), Some(Default::default())); - let mut block = latest_block; - loop { - if fast_synced_block == block { - break; - } + assert_eq!( + alice.db.block_header(fast_synced_block), + bob.db.block_header(fast_synced_block) + ); + assert_eq!( + alice.db.block_events(fast_synced_block), + bob.db.block_events(fast_synced_block) + ); + assert_eq!( + alice.db.block_synced(fast_synced_block), + bob.db.block_synced(fast_synced_block) + ); - log::trace!("assert block {block}"); + latest_mb + }; - // Check block meta, excluding codes_queue, which can vary. - let alice_meta = alice.db.block_meta(block); - let bob_meta = bob.db.block_meta(block); - assert!( - alice_meta.prepared && bob_meta.prepared, - "Block {block} is not prepared for alice or bob" - ); - assert_eq!( - alice_meta.last_committed_batch, - bob_meta.last_committed_batch + let assert_chain = + |latest_block, fast_synced_block, fast_synced_mb, alice: &Node, bob: &Node| { + log::info!("Assert chain in range {latest_block}..{fast_synced_block}"); + assert_ne!( + latest_block, fast_synced_block, + "post-fast-sync comparison must cover at least one block" ); - assert_eq!(alice_meta.last_committed_mb, bob_meta.last_committed_mb); - assert_eq!(alice_meta.last_committed_eb, bob_meta.last_committed_eb); + + IntegrityVerifier::new(alice.db.clone()) + .verify_chain(latest_block, fast_synced_block) + .expect("failed to verify Alice database"); + + let alice_globals = alice.db.globals(); + let bob_globals = bob.db.globals(); assert_eq!( - alice_meta.latest_era_validators_committed, - bob_meta.latest_era_validators_committed + alice_globals.latest_prepared_eb_hash, + bob_globals.latest_prepared_eb_hash ); - assert_eq!(alice.db.block_header(block), bob.db.block_header(block)); - assert_eq!(alice.db.block_events(block), bob.db.block_events(block)); - assert_eq!(alice.db.block_synced(block), bob.db.block_synced(block)); + let mut block = latest_block; + loop { + if fast_synced_block == block { + break; + } - let header = alice.db.block_header(block).unwrap(); - block = header.parent_hash; - } - }; + log::trace!("assert block {block}"); + + // Check block meta, excluding fast-sync-derived boundary fields + // that can differ in the restored chain prefix. + let alice_meta = alice.db.block_meta(block); + let bob_meta = bob.db.block_meta(block); + assert!( + alice_meta.prepared && bob_meta.prepared, + "Block {block} is not prepared for Alice or Bob" + ); + assert_eq!( + alice_meta.latest_era_validators_committed, + bob_meta.latest_era_validators_committed + ); + + if let Some(mb_hash) = alice_meta + .last_committed_mb + .filter(|mb_hash| Some(*mb_hash) == bob_meta.last_committed_mb) + && alice.db.mb_compact_block(mb_hash).is_some() + && bob.db.mb_compact_block(mb_hash).is_some() + && alice.db.mb_meta(mb_hash).computed == bob.db.mb_meta(mb_hash).computed + { + assert_eq!( + alice.db.mb_program_states(mb_hash), + bob.db.mb_program_states(mb_hash) + ); + assert_eq!(alice.db.mb_schedule(mb_hash), bob.db.mb_schedule(mb_hash)); + if mb_hash != fast_synced_mb { + assert_eq!(alice.db.mb_outcome(mb_hash), bob.db.mb_outcome(mb_hash)); + } + } + + assert_eq!(alice.db.block_header(block), bob.db.block_header(block)); + assert_eq!(alice.db.block_events(block), bob.db.block_events(block)); + assert_eq!(alice.db.block_synced(block), bob.db.block_synced(block)); + + let header = alice.db.block_header(block).unwrap(); + block = header.parent_hash; + } + }; let config = TestEnvConfig { validators: ValidatorsConfig::PreDefined(3), @@ -3300,26 +3390,35 @@ async fn fast_sync() { assert!(!fast_sync_target.is_zero()); test_info!("Fast-sync source MB {alice_mb}, target EB {fast_sync_target}"); - let source_bootstrap_address = validators[0].multiaddr.clone().unwrap(); - test_info!("Starting Bob with fast sync"); - let mut bob = env - .new_node( - NodeConfig::named("Bob") - .fast_sync() - .network_bootstrap_address(source_bootstrap_address), - ) - .await; + let mut bob = env.new_node(NodeConfig::named("Bob").fast_sync()).await; tokio::time::timeout(Duration::from_secs(45), bob.start_service()) .await .expect("Bob did not finish fast sync"); let fast_synced_block = bob.latest_fast_synced_block.take().unwrap(); - assert_eq!(fast_synced_block, fast_sync_target); + assert!(!fast_synced_block.is_zero()); + let fast_synced_mb = assert_fast_sync_boundary(fast_synced_block, &validators[0], &bob); + + test_info!("Sending post-fast-sync message and comparing Bob with Alice"); + let mut alice_events = validators[0].events(); + let mut bob_events = bob.events(); + let reply = env + .send_message(ping_id, b"PING") + .await + .unwrap() + .wait_for() + .await + .unwrap(); + assert_eq!(reply.payload, b"PONG"); + + let latest_block = env.latest_block().await.hash; + wait_till_block_prepared(&mut alice_events, latest_block).await; + wait_till_block_prepared(&mut bob_events, latest_block).await; assert_chain( - fast_sync_target, + latest_block, fast_synced_block, - alice_mb, + fast_synced_mb, &validators[0], &bob, ); diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index 5bd6468fcc9..af463d9bae0 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -457,7 +457,6 @@ impl TestEnv { validator_config, rpc: service_rpc_config, fast_sync, - network_bootstrap_addresses: custom_network_bootstrap_addresses, } = config; let db = match db { @@ -478,11 +477,6 @@ impl TestEnv { (format!("/memory/{nonce}"), bootstrap_address.clone()) }) .unzip(); - let network_bootstrap_addresses = if custom_network_bootstrap_addresses.is_empty() { - network_bootstrap_address.into_iter().collect() - } else { - custom_network_bootstrap_addresses - }; let network_public_key = network_address.as_ref().map(|_| { self.signer .generate() @@ -521,7 +515,7 @@ impl TestEnv { validator_config, network_public_key, network_address, - network_bootstrap_addresses, + network_bootstrap_address, service_rpc_config, fast_sync, commitment_delay_limit: self.commitment_delay_limit, @@ -919,8 +913,6 @@ pub struct NodeConfig { pub rpc: Option, /// Do P2P database synchronization before the main loop pub fast_sync: bool, - /// Override bootstrap peer address for tests that need a specific data-serving peer. - pub network_bootstrap_addresses: Vec, } impl NodeConfig { @@ -960,11 +952,6 @@ impl NodeConfig { self.fast_sync = true; self } - - pub fn network_bootstrap_address(mut self, address: impl Into) -> Self { - self.network_bootstrap_addresses.push(address.into()); - self - } } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -1032,7 +1019,7 @@ pub struct Node { validator_config: Option, network_public_key: Option, network_address: Option, - network_bootstrap_addresses: Vec, + network_bootstrap_address: Option, service_rpc_config: Option, fast_sync: bool, commitment_delay_limit: std::num::NonZero, @@ -1236,7 +1223,7 @@ impl Node { .expect("failed to create blob loader") .into_box(); - let wait_for_network = !self.network_bootstrap_addresses.is_empty(); + let wait_for_network = self.network_bootstrap_address.is_some(); let network = self.construct_network_service(latest_block, latest_validators); if let Some(addr) = self.network_address.as_ref() { @@ -1389,11 +1376,9 @@ impl Node { let mut config = NetworkConfig::new_test(network_key, self.eth_cfg.router_address); config.listen_addresses = [multiaddr.clone()].into(); config.external_addresses = [multiaddr.clone()].into(); - config.bootstrap_addresses = self - .network_bootstrap_addresses - .iter() - .map(|addr| addr.parse().unwrap()) - .collect(); + if let Some(bootstrap_addr) = self.network_bootstrap_address.as_ref() { + config.bootstrap_addresses = [bootstrap_addr.parse().unwrap()].into(); + } let runtime_config = NetworkRuntimeConfig { latest_block_header: latest_block.header, From 201aba01cb2e0971965728788ec80faa9208712c Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 4 Jun 2026 19:45:36 +0300 Subject: [PATCH 14/72] Manual fixes --- ethexe/service/src/fast_sync.rs | 77 +++++++++++++++++++++++---------- ethexe/service/src/lib.rs | 13 +----- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index e6cd629711a..72cfc66920c 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -181,21 +181,31 @@ enum RequestMetadata { Mailbox, UserMailbox, DispatchStash, + /// Any data we only insert into the database. Data, } impl RequestMetadata { fn is_data(self) -> bool { - matches!(self, Self::Data) + matches!(self, RequestMetadata::Data) } } #[derive(Debug)] struct RequestManager { db: Database, + + /// Total completed requests total_completed_requests: u64, + /// Total pending requests total_pending_requests: u64, + + /// Pending requests are either: + /// * Skipped if they are `RequestMetadata::Data` and exist in the database + /// * Completed if the database has keys + /// * Converted into one network request pending_requests: HashMap, + /// Completed requests responses: Vec<(RequestMetadata, Vec)>, } @@ -211,7 +221,11 @@ impl RequestManager { } fn add(&mut self, hash: H256, metadata: RequestMetadata) { - debug_assert_ne!(hash, H256::zero(), "zero hash cannot be requested"); + debug_assert_ne!( + hash, + H256::zero(), + "zero hash is cannot be requested from db or network" + ); let old_metadata = self.pending_requests.insert(hash, metadata); @@ -277,21 +291,20 @@ impl RequestManager { .expect("unknown pending request"); let db_hash = self.db.cas().write(&data); - ensure!( - hash == db_hash, - "bitswap returned data with unexpected hash" - ); + debug_assert_eq!(hash, db_hash); self.responses.push((metadata, data)); } - debug_assert!( - pending_network_requests.is_empty(), - "network service must gather all requested hashes" + debug_assert_eq!( + pending_network_requests, + HashMap::new(), + "network service guarantees it gathers all hashes" ); Ok(()) } + /// (total completed request, total pending requests) fn stats(&self) -> (u64, u64) { let completed = self.total_completed_requests; let pending = self.total_pending_requests; @@ -357,9 +370,16 @@ impl Drop for RequestManager { fn drop(&mut self) { #[cfg(debug_assertions)] { - assert_eq!(self.total_completed_requests, self.total_pending_requests); - assert!(self.pending_requests.is_empty()); - assert!(self.responses.is_empty()); + let Self { + db: _, + total_completed_requests, + total_pending_requests, + pending_requests, + responses, + } = self; + assert_eq!(total_completed_requests, total_pending_requests); + assert_eq!(*pending_requests, HashMap::new()); + assert_eq!(*responses, Vec::new()); } } } @@ -388,7 +408,6 @@ async fn sync_latest_mb( mb_hash: H256, code_ids: &BTreeSet, program_state_hashes: BTreeMap, - block_height: u32, ) -> Result { restore_compact_mb(db, mb_hash); @@ -489,14 +508,6 @@ async fn sync_latest_mb( }) .collect(); - let schedule = ScheduleRestorer::from_storage(db, &program_states, block_height)?.restore(); - db.set_mb_program_states(mb_hash, program_states.clone()); - db.set_mb_schedule(mb_hash, schedule); - // The committed fast-sync anchor is never replayed for batch creation, so - // keep queue-derived outcome invariants out of restoration and persist an - // empty row only to satisfy database walks over committed MB metadata. - db.set_mb_outcome(mb_hash, Default::default()); - Ok(program_states) } @@ -515,7 +526,7 @@ async fn instrument_codes( for &code_id in &code_ids { let original_code = db .original_code(code_id) - .with_context(|| format!("code {code_id:?} was not fetched from the network"))?; + .expect("`sync_from_network` must fulfill database"); compute.process_code(CodeAndIdUnchecked { code_id, code: original_code, @@ -542,6 +553,7 @@ async fn set_tx_pool_data_requirement( let to = latest_committed_block_height as u64; let from = to.saturating_sub(injected::VALIDITY_WINDOW as u64); + // TODO: #4926 unsafe solution - we need it for taking events from predecessor blocks in ethexe-compute let blocks = block_loader.load_many(from..=to).await?; for BlockData { hash, @@ -585,6 +597,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { observer, compute, network, + malachite, db, #[cfg(test)] sender, @@ -667,13 +680,12 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { db.set_program_code_id(program_id, code_id); } - sync_latest_mb( + let program_states = sync_latest_mb( network, db, latest_committed_mb, &code_ids, program_state_hashes, - header.height, ) .await?; db.mutate_mb_meta(latest_committed_mb, |meta| { @@ -702,6 +714,14 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { }, ); + let schedule = ScheduleRestorer::from_storage(db, &program_states, header.height)?.restore(); + db.set_mb_program_states(latest_committed_mb, program_states.clone()); + db.set_mb_schedule(latest_committed_mb, schedule); + // The committed fast-sync anchor is never replayed for batch creation, so + // keep queue-derived outcome invariants out of restoration and persist an + // empty row only to satisfy database walks over committed MB metadata. + db.set_mb_outcome(latest_committed_mb, Default::default()); + db.globals_mutate(|globals| { globals.start_block_hash = block_hash; globals.latest_synced_eb = SimpleBlockData { @@ -713,6 +733,15 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { globals.latest_computed_mb_hash = latest_committed_mb; }); + if !block_hash.is_zero() + && let Some(malachite) = malachite.as_mut() + { + malachite.receive_new_chain_head(SimpleBlockData { + hash: block_hash, + header, + }); + } + log::info!( "Fast synchronization done: synced to {block_hash:?}, height {:?}, MB {latest_committed_mb}", header.height diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 73a15911883..281bd71770b 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -42,7 +42,7 @@ use anyhow::{Context, Result, bail}; use ethexe_blob_loader::{BlobLoader, BlobLoaderEvent, BlobLoaderService, ConsensusLayerConfig}; use ethexe_common::{ CodeAndIdUnchecked, PromiseEmissionMode, - db::{GlobalsStorageRO, GlobalsStorageRW, MbStorageRO, OnChainStorageRO}, + db::{GlobalsStorageRW, MbStorageRO, OnChainStorageRO}, injected::{CompactPromise, Receipt}, network::VerifiedValidatorMessage, }; @@ -581,17 +581,6 @@ impl Service { pub async fn run(mut self) -> Result<()> { if self.fast_sync { fast_sync::sync(&mut self).await?; - - let block_hash = self.db.globals().latest_prepared_eb_hash; - if !block_hash.is_zero() { - if let Some(malachite) = self.malachite.as_mut() { - let block = self - .db - .block_simple_data(block_hash) - .context("Cannot find fast-synced chain head")?; - malachite.receive_new_chain_head(block); - } - } } self.run_inner().await.inspect_err(|err| { From 599a385f4ccbaa2e40d27aaf006db9c4b64051a4 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 4 Jun 2026 20:01:27 +0300 Subject: [PATCH 15/72] Simplify request and response in bitswap --- ethexe/network/src/bitswap.rs | 90 +++++++++------------------------ ethexe/network/src/lib.rs | 29 +---------- ethexe/service/src/fast_sync.rs | 19 +++++-- 3 files changed, 40 insertions(+), 98 deletions(-) diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs index 2ba2940c042..9b772fdd946 100644 --- a/ethexe/network/src/bitswap.rs +++ b/ethexe/network/src/bitswap.rs @@ -46,41 +46,6 @@ use tokio::{ const BLAKE2B_CODE: u64 = 0xb220; // standard BLAKE2b multihash code const RAW_CODEC: u64 = 0x55; // standard CID raw codec -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub enum Request { - Hash(H256), -} - -impl From for Request { - fn from(hash: H256) -> Self { - Self::Hash(hash) - } -} - -impl Request { - fn into_cid(self) -> Cid { - let (codec, hash) = match self { - Request::Hash(hash) => (RAW_CODEC, hash), - }; - - Cid::new_v1( - codec, - Multihash::wrap(BLAKE2B_CODE, hash.as_bytes()).expect("size is always correct"), - ) - } - - fn into_response(self, data: Vec) -> Response { - match self { - Request::Hash(_) => Response::Hash(data), - } - } -} - -#[derive(Debug, Clone, Eq, PartialEq, derive_more::Unwrap)] -pub enum Response { - Hash(Vec), -} - #[derive(Debug, Clone, Copy, Default)] pub struct Config { /// Restart stalled requests after some time. @@ -102,16 +67,14 @@ impl Config { #[derive(Clone)] pub struct Handle { - inner: mpsc::UnboundedSender<(Request, oneshot::Sender)>, + inner: mpsc::UnboundedSender<(H256, oneshot::Sender>)>, auto_retry: bool, } impl Handle { const RETRY_TIMEOUT: Duration = Duration::from_secs(5); - pub async fn request(&self, request: impl Into) -> Response { - let request = request.into(); - + pub async fn request(&self, request: H256) -> Vec { if !self.auto_retry { return self.inner_request(request).await; } @@ -126,7 +89,7 @@ impl Handle { } } - async fn inner_request(&self, request: Request) -> Response { + async fn inner_request(&self, request: H256) -> Vec { let (tx, rx) = oneshot::channel(); self.inner @@ -255,8 +218,8 @@ type InnerBehaviour = beetswap::Behaviour<32, Blockstore>; pub struct Behaviour { inner: InnerBehaviour, handle: Handle, - rx: mpsc::UnboundedReceiver<(Request, oneshot::Sender)>, - requests: HashMap)>, + rx: mpsc::UnboundedReceiver<(H256, oneshot::Sender>)>, + requests: HashMap>>, } impl Behaviour { @@ -283,12 +246,18 @@ impl Behaviour { self.handle.clone() } + fn create_cid(hash: H256) -> Cid { + Cid::new_v1( + RAW_CODEC, + Multihash::wrap(BLAKE2B_CODE, hash.as_bytes()).expect("size is always correct"), + ) + } + fn handle_inner_event(&mut self, event: beetswap::Event) { match event { beetswap::Event::GetQueryResponse { query_id, data } => { - if let Some((request, channel)) = self.requests.remove(&query_id) { - let response = request.into_response(data); - let _ = channel.send(response); + if let Some(channel) = self.requests.remove(&query_id) { + let _ = channel.send(data); } } beetswap::Event::GetQueryError { query_id, error } => { @@ -379,7 +348,7 @@ impl NetworkBehaviour for Behaviour { &mut self, cx: &mut Context<'_>, ) -> Poll>> { - self.requests.retain(|&query_id, (_, channel)| { + self.requests.retain(|&query_id, channel| { if channel.is_closed() { self.inner.cancel(query_id); return false; @@ -389,9 +358,9 @@ impl NetworkBehaviour for Behaviour { }); while let Poll::Ready(Some((request, channel))) = self.rx.poll_recv(cx) { - let cid = request.into_cid(); + let cid = Self::create_cid(request); let query_id = self.inner.get(&cid); - self.requests.insert(query_id, (request, channel)); + self.requests.insert(query_id, channel); } if let Poll::Ready(to_swarm) = self.inner.poll(cx) { @@ -433,21 +402,12 @@ mod tests { } } - #[test] - fn request_converts_to_expected_cid() { - let hash = H256::from([1; 32]); - let cid = Request::Hash(hash).into_cid(); - assert_eq!(cid.codec(), RAW_CODEC); - assert_eq!(cid.hash().code(), BLAKE2B_CODE); - assert_eq!(cid.hash().digest(), hash.as_bytes()); - } - #[tokio::test] async fn blockstore_reads_raw_data() { let db = ethexe_db::Database::memory(); let hash = db.cas().write(b"hello"); let blockstore = Blockstore { db: Box::new(db) }; - let cid = Request::Hash(hash).into_cid(); + let cid = Behaviour::create_cid(hash); let data = blockstore.get(&cid).await.unwrap(); @@ -493,7 +453,7 @@ mod tests { .cas() .write(&vec![0; Blockstore::MAX_BLOCK_SIZE as usize + 1]); let blockstore = Blockstore { db: Box::new(db) }; - let cid = Request::Hash(hash).into_cid(); + let cid = Behaviour::create_cid(hash); let error = blockstore.get(&cid).await.unwrap_err(); @@ -505,7 +465,7 @@ mod tests { let blockstore = Blockstore { db: Box::new(PanickingDatabase), }; - let cid = Request::Hash(H256::from([6; 32])).into_cid(); + let cid = Behaviour::create_cid(H256::from([6; 32])); let error = blockstore.get(&cid).await.unwrap_err(); @@ -549,18 +509,16 @@ mod tests { let pending = tokio::spawn(async move { handle.request(hash).await }); let (request, first_response) = rx.recv().await.unwrap(); - assert_eq!(request, Request::Hash(hash)); + assert_eq!(request, hash); time::advance(Handle::RETRY_TIMEOUT).await; let (request, second_response) = rx.recv().await.unwrap(); - assert_eq!(request, Request::Hash(hash)); + assert_eq!(request, hash); assert!(first_response.is_closed()); - second_response - .send(Response::Hash(b"hello".to_vec())) - .unwrap(); + second_response.send(b"hello".to_vec()).unwrap(); let response = pending.await.unwrap(); - assert_eq!(response, Response::Hash(b"hello".to_vec())); + assert_eq!(response, b"hello".to_vec()); } } diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index 5ec87e4f625..5ba1af1549d 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -623,25 +623,6 @@ impl NetworkService { self.swarm.behaviour().bitswap.handle() } - async fn bitswap_request(&mut self, request: bitswap::Request) -> bitswap::Response { - let bitswap = self.bitswap_handle(); - let fut = bitswap.request(request); - tokio::pin!(fut); - - loop { - tokio::select! { - _ = self.select_next_some() => {}, - response = &mut fut => break response, - } - } - } - - pub async fn bitswap_fetch_hash(&mut self, hash: H256) -> Vec { - let bitswap::Response::Hash(data) = - self.bitswap_request(bitswap::Request::Hash(hash)).await; - data - } - /// Refresh validator-era state after the chain head changes. /// /// This updates both validator-message verification and validator @@ -968,13 +949,7 @@ mod tests { ) .await .expect("time has elapsed"); - assert_eq!( - response, - ( - bitswap::Response::Hash(b"hello".to_vec()), - bitswap::Response::Hash(b"world".to_vec()) - ) - ); + assert_eq!(response, (b"hello".to_vec(), b"world".to_vec())); } #[tokio::test] @@ -998,7 +973,7 @@ mod tests { let response = timeout(Duration::from_secs(5), service1_handle.request(hello)) .await .expect("time has elapsed"); - assert_eq!(response, bitswap::Response::Hash(b"hello".to_vec())); + assert_eq!(response, b"hello".to_vec()); } #[tokio::test] diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 72cfc66920c..af0adb1f7c1 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -40,7 +40,7 @@ use ethexe_runtime_common::{ UserMailbox, Waitlist, }, }; -use futures::StreamExt; +use futures::{FutureExt, StreamExt, stream::FuturesUnordered}; use gprimitives::{ActorId, CodeId, H256}; use parity_scale_codec::Decode; use std::{ @@ -243,10 +243,19 @@ impl RequestManager { let pending_network_requests = self.handle_pending_requests(); if !pending_network_requests.is_empty() { - let mut response = Vec::with_capacity(pending_network_requests.len()); - for &hash in pending_network_requests.keys() { - response.push((hash, network.bitswap_fetch_hash(hash).await)); - } + let bitswap_handle = network.bitswap_handle(); + let mut request = pending_network_requests + .keys() + .map(|&hash| bitswap_handle.request(hash).map(move |data| (hash, data))) + .collect::>() + .collect::>(); + + let response = loop { + tokio::select! { + _ = network.select_next_some() => {}, + response = &mut request => break response, + } + }; self.handle_response(pending_network_requests, response)?; } From 67e27c5ee27b208d837c074d76d177bbd9edad0d Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 4 Jun 2026 20:17:16 +0300 Subject: [PATCH 16/72] Implement `Handle::request_many` --- ethexe/network/src/bitswap.rs | 11 ++++++++++- ethexe/service/src/fast_sync.rs | 13 ++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs index 9b772fdd946..9da27c708b4 100644 --- a/ethexe/network/src/bitswap.rs +++ b/ethexe/network/src/bitswap.rs @@ -20,7 +20,7 @@ use beetswap::multihasher::{Multihasher, MultihasherError}; use blockstore::{block::CidError, cond_send::CondSend}; use cid::{Cid, CidGeneric}; use ethexe_common::db::HashStorageRO; -use futures::FutureExt; +use futures::{FutureExt, Stream, stream::FuturesUnordered}; use gprimitives::H256; use libp2p::{ Multiaddr, PeerId, @@ -89,6 +89,15 @@ impl Handle { } } + pub fn request_many<'a>( + &'a self, + iter: impl Iterator + 'a, + ) -> impl Stream)> + 'a { + FuturesUnordered::from_iter( + iter.map(|hash| self.request(hash).map(move |data| (hash, data))), + ) + } + async fn inner_request(&self, request: H256) -> Vec { let (tx, rx) = oneshot::channel(); diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index af0adb1f7c1..465d326d9f7 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -40,7 +40,7 @@ use ethexe_runtime_common::{ UserMailbox, Waitlist, }, }; -use futures::{FutureExt, StreamExt, stream::FuturesUnordered}; +use futures::StreamExt; use gprimitives::{ActorId, CodeId, H256}; use parity_scale_codec::Decode; use std::{ @@ -243,12 +243,10 @@ impl RequestManager { let pending_network_requests = self.handle_pending_requests(); if !pending_network_requests.is_empty() { - let bitswap_handle = network.bitswap_handle(); - let mut request = pending_network_requests - .keys() - .map(|&hash| bitswap_handle.request(hash).map(move |data| (hash, data))) - .collect::>() - .collect::>(); + let bitswap = network.bitswap_handle(); + let mut request = bitswap + .request_many(pending_network_requests.keys().copied()) + .collect(); let response = loop { tokio::select! { @@ -256,6 +254,7 @@ impl RequestManager { response = &mut request => break response, } }; + drop(request); self.handle_response(pending_network_requests, response)?; } From 23a5bae53777c331371c18e06c3b5866de5d705c Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Fri, 5 Jun 2026 14:38:24 +0300 Subject: [PATCH 17/72] Manual fixes --- ethexe/service/src/fast_sync.rs | 150 ++++++++++++++++---------------- 1 file changed, 73 insertions(+), 77 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 465d326d9f7..f7af335d160 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -11,7 +11,7 @@ use ethexe_common::{ StateHashWithQueueSize, db::{ BlockMetaStorageRO, CodesStorageRO, CodesStorageRW, CompactMb, ConfigStorageRO, - GlobalsStorageRW, MbStorageRO, MbStorageRW, OnChainStorageRW, PreparedBlockData, + GlobalsStorageRW, MbStorageRW, OnChainStorageRW, PreparedBlockData, }, events::{ BlockEvent, RouterEvent, @@ -50,7 +50,7 @@ use std::{ struct EventData { latest_committed_batch: Digest, - committed_mbs: Vec, + latest_committed_mb: H256, latest_committed_eb: Option, } @@ -61,7 +61,7 @@ impl EventData { highest_block: H256, ) -> Result> { let mut latest_committed_batch = None; - let mut committed_mbs = Vec::new(); + let mut latest_committed_mb = None; let mut latest_committed_eb = None; let mut block = highest_block; @@ -72,16 +72,14 @@ impl EventData { match event { BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { digest, - })) if latest_committed_batch.is_none() => { - latest_committed_batch = Some(*digest); + })) => { + latest_committed_batch.get_or_insert(*digest); } BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb_hash))) => { - committed_mbs.push(*mb_hash); + latest_committed_mb.get_or_insert(*mb_hash); } - BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(eb_hash))) - if latest_committed_eb.is_none() => - { - latest_committed_eb = Some(*eb_hash); + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(eb_hash))) => { + latest_committed_eb.get_or_insert(*eb_hash); } _ => {} } @@ -93,13 +91,13 @@ impl EventData { } } - if committed_mbs.is_empty() { + let Some(latest_committed_mb) = latest_committed_mb else { return Ok(None); - } + }; Ok(Some(Self { latest_committed_batch: latest_committed_batch.unwrap_or_default(), - committed_mbs, + latest_committed_mb, latest_committed_eb, })) } @@ -239,7 +237,7 @@ impl RequestManager { async fn request( &mut self, network: &mut NetworkService, - ) -> Result)>>> { + ) -> Option)>> { let pending_network_requests = self.handle_pending_requests(); if !pending_network_requests.is_empty() { @@ -256,16 +254,16 @@ impl RequestManager { }; drop(request); - self.handle_response(pending_network_requests, response)?; + self.handle_response(pending_network_requests, response); } let continue_processing = !(self.pending_requests.is_empty() && self.responses.is_empty()); if continue_processing { - let responses = self.responses.drain(..).collect::>(); + let responses: Vec<_> = self.responses.drain(..).collect(); self.total_completed_requests += responses.len() as u64; - Ok(Some(responses)) + Some(responses) } else { - Ok(None) + None } } @@ -292,7 +290,7 @@ impl RequestManager { &mut self, mut pending_network_requests: HashMap, data: Vec<(H256, Vec)>, - ) -> Result<()> { + ) { for (hash, data) in data { let metadata = pending_network_requests .remove(&hash) @@ -309,7 +307,6 @@ impl RequestManager { HashMap::new(), "network service guarantees it gathers all hashes" ); - Ok(()) } /// (total completed request, total pending requests) @@ -392,54 +389,45 @@ impl Drop for RequestManager { } } -fn restore_compact_mb(db: &Database, mb_hash: H256) { - if db.mb_compact_block(mb_hash).is_some() { - return; - } - - let transactions_hash = db.set_transactions(Transactions::default()); - // The committed MB is a fast-sync boundary: chain data gives us the head - // hash, but not the Malachite envelope `(parent, height, payload_hash)`. - // Store a synthetic compact row so DB walks have an anchor without - // pretending to restore the pre-boundary MB chain. - let compact = CompactMb { - parent: H256::zero(), - height: 0, - transactions_hash, - }; - db.set_mb_compact_block(mb_hash, compact); -} - -async fn sync_latest_mb( +/// Synchronize program states and related data from the network. +/// +/// This asynchronous function fetches data from the network based on program +/// state hashes and associated metadata using a request-manager mechanism. It also enriches +/// the program states with cached queue sizes. +async fn sync_from_network( network: &mut NetworkService, db: &Database, - mb_hash: H256, code_ids: &BTreeSet, - program_state_hashes: BTreeMap, -) -> Result { - restore_compact_mb(db, mb_hash); - + program_states: BTreeMap, +) -> ProgramStates { let mut restored_cached_queue_sizes = BTreeMap::new(); + let mut manager = RequestManager::new(db.clone()); - for &state_hash in program_state_hashes.values() { - manager.add(state_hash, RequestMetadata::ProgramState); + + for &state in program_states.values() { + manager.add(state, RequestMetadata::ProgramState); } + for &code_id in code_ids { manager.add(code_id.into(), RequestMetadata::Data); } - while let Some(responses) = manager.request(network).await? { + loop { let (completed, pending) = manager.stats(); log::info!("[{completed:>05} / {pending:>05}] Getting network data"); + let Some(responses) = manager.request(network).await else { + break; + }; + for (metadata, data) in responses { match metadata { RequestMetadata::ProgramState => { let state: ProgramState = Decode::decode(&mut &data[..]).expect("bitswap must validate data"); - let state_hash = ethexe_db::hash(&data); + let program_state_hash = ethexe_db::hash(&data); restored_cached_queue_sizes.insert( - state_hash, + program_state_hash, ( state.canonical_queue.cached_queue_size, state.injected_queue.cached_queue_size, @@ -492,14 +480,15 @@ async fn sync_latest_mb( Decode::decode(&mut &data[..]).expect("bitswap must validate data"); ethexe_db::visitor::walk(&mut manager, DispatchStashNode { dispatch_stash }); } - RequestMetadata::Data => {} + RequestMetadata::Data => continue, } } } log::info!("Network data getting is done"); - let program_states = program_state_hashes + // Enrich program states with cached queue size + program_states .into_iter() .map(|(program_id, hash)| { let (canonical_queue_size, injected_queue_size) = *restored_cached_queue_sizes @@ -514,11 +503,10 @@ async fn sync_latest_mb( }, ) }) - .collect(); - - Ok(program_states) + .collect() } +/// Instruments a set of codes by delegating their processing to the `ComputeService`. async fn instrument_codes( compute: &mut ComputeService, db: &Database, @@ -620,6 +608,9 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { let finalized_block = observer .block_loader() + // we get finalized block to avoid block reorganization + // because we restore the database only for the latest block of a chain, + // and thus the reorganization can lead us to an empty block .load_simple(BlockId::finalized()) .await .context("failed to get latest finalized block")? @@ -660,15 +651,10 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { } }; - let latest_committed_mb = event_data - .committed_mbs - .first() - .copied() - .expect("EventData always contains at least one committed MB"); let EventData { latest_committed_batch, + latest_committed_mb, latest_committed_eb, - .. } = event_data; let latest_committed_eb = latest_committed_eb.unwrap_or(genesis_block_hash); @@ -684,27 +670,18 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { let program_state_hashes = collect_program_state_hashes(observer, state_query_block, &program_code_ids).await?; - for (&program_id, &code_id) in &program_code_ids { - db.set_program_code_id(program_id, code_id); - } - - let program_states = sync_latest_mb( - network, - db, - latest_committed_mb, - &code_ids, - program_state_hashes, - ) - .await?; - db.mutate_mb_meta(latest_committed_mb, |meta| { - meta.computed = true; - meta.last_advanced_eb = latest_committed_eb; - }); + let program_states = sync_from_network(network, db, &code_ids, program_state_hashes).await; instrument_codes(compute, db, code_ids).await?; + let schedule = ScheduleRestorer::from_storage(db, &program_states, header.height)?.restore(); + set_tx_pool_data_requirement(db, &block_loader, header.height).await?; + for (program_id, code_id) in program_code_ids { + db.set_program_code_id(program_id, code_id); + } + let latest_era_with_committed_validators = latest_era_with_committed_validators(db, &observer.router_query(), block_hash).await?; @@ -715,14 +692,33 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { header, events, latest_era_with_committed_validators, + // NOTE: there is no invariant that fast sync should recover codes queue codes_queue: Default::default(), + // TODO #4812: using `latest_committed_batch` here is not correct, + // because `latest_committed_batch` is latest for finalized block, not for `block_hash`. last_committed_batch: latest_committed_batch, last_committed_mb: latest_committed_mb, last_committed_eb: latest_committed_eb, }, ); - let schedule = ScheduleRestorer::from_storage(db, &program_states, header.height)?.restore(); + db.mutate_mb_meta(latest_committed_mb, |meta| { + meta.computed = true; + meta.last_advanced_eb = latest_committed_eb; + }); + + let transactions_hash = db.set_transactions(Transactions::default()); + // The committed MB is a fast-sync boundary: chain data gives us the head + // hash, but not the Malachite envelope `(parent, height, payload_hash)`. + // Store a synthetic compact row so DB walks have an anchor without + // pretending to restore the pre-boundary MB chain. + let compact = CompactMb { + parent: H256::zero(), + height: 0, + transactions_hash, + }; + db.set_mb_compact_block(latest_committed_mb, compact); + db.set_mb_program_states(latest_committed_mb, program_states.clone()); db.set_mb_schedule(latest_committed_mb, schedule); // The committed fast-sync anchor is never replayed for batch creation, so From 25a036621019cefb15593b1d6ddb2a3338db7191 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Sun, 7 Jun 2026 20:57:49 +0300 Subject: [PATCH 18/72] Speed up Malachite sync in tests --- ethexe/malachite/core/src/config.rs | 9 +++++++++ ethexe/malachite/core/src/lib.rs | 5 ++++- ethexe/malachite/core/src/service.rs | 14 ++++++++++++-- ethexe/malachite/core/tests/multi_validators.rs | 1 + ethexe/malachite/service/src/config.rs | 4 ++++ ethexe/malachite/service/src/lib.rs | 6 ++++-- ethexe/malachite/service/src/service.rs | 1 + .../malachite/service/tests/restart_resilience.rs | 1 + ethexe/service/src/tests/utils/env.rs | 6 ++++-- 9 files changed, 40 insertions(+), 7 deletions(-) diff --git a/ethexe/malachite/core/src/config.rs b/ethexe/malachite/core/src/config.rs index f97e716074f..6801b888884 100644 --- a/ethexe/malachite/core/src/config.rs +++ b/ethexe/malachite/core/src/config.rs @@ -44,6 +44,13 @@ pub enum NodeRole { FullNode, } +#[derive(Debug, Copy, Clone, Default)] +pub enum Environment { + #[default] + Production, + Test, +} + /// All configuration the service needs to bootstrap the malachite /// engine. /// @@ -51,6 +58,8 @@ pub enum NodeRole { /// live behind [`crate::Externalities`] — they don't belong here. #[derive(Clone, Debug)] pub struct MalachiteConfig { + pub env: Environment, + /// Local libp2p listen address. pub listen_addr: SocketAddr, diff --git a/ethexe/malachite/core/src/lib.rs b/ethexe/malachite/core/src/lib.rs index a67f5bc996e..4749188e2e4 100644 --- a/ethexe/malachite/core/src/lib.rs +++ b/ethexe/malachite/core/src/lib.rs @@ -78,7 +78,10 @@ mod store; mod streaming; pub use crate::{ - config::{MalachiteConfig, Multiaddr, NodeRole, ValidatorEntry}, + config::{ + Environment as MalachiteConfigEnvironment, MalachiteConfig, Multiaddr, NodeRole, + ValidatorEntry, + }, externalities::{BlockPayload, Externalities}, service::{MService, MalachiteService}, signing::{ diff --git a/ethexe/malachite/core/src/service.rs b/ethexe/malachite/core/src/service.rs index 9c0bb8dd47e..d5adb8d104e 100644 --- a/ethexe/malachite/core/src/service.rs +++ b/ethexe/malachite/core/src/service.rs @@ -6,7 +6,7 @@ use crate::{ app, codec::ScaleCodec, - config::{MalachiteConfig, NodeRole}, + config::{Environment, MalachiteConfig, NodeRole}, context::{MalachiteCtx, Validator, ValidatorSet}, externalities::{BlockPayload, Externalities}, signing::{ @@ -345,10 +345,20 @@ fn build_inner_config(cfg: &MalachiteConfig, moniker: &str) -> InnerNodeConfig { ..Default::default() }, }; + let value_sync = match cfg.env { + Environment::Production => ValueSyncConfig::default(), + Environment::Test => ValueSyncConfig { + status_update_interval: Duration::from_millis(500), + request_timeout: Duration::from_secs(3), + parallel_requests: 16, + batch_size: 32, + ..Default::default() + }, + }; InnerNodeConfig { moniker: moniker.to_string(), consensus, - value_sync: ValueSyncConfig::default(), + value_sync, logging: LoggingConfig::default(), metrics: MetricsConfig::default(), runtime: RuntimeConfig::default(), diff --git a/ethexe/malachite/core/tests/multi_validators.rs b/ethexe/malachite/core/tests/multi_validators.rs index c4119ad1646..6c7330394f3 100644 --- a/ethexe/malachite/core/tests/multi_validators.rs +++ b/ethexe/malachite/core/tests/multi_validators.rs @@ -295,6 +295,7 @@ fn build_config_with_role( role: NodeRole, ) -> MalachiteConfig { MalachiteConfig { + env: Default::default(), listen_addr: setup.listen_addr, base: setup.home.path().to_path_buf(), persistent_peers: peers, diff --git a/ethexe/malachite/service/src/config.rs b/ethexe/malachite/service/src/config.rs index 49a0d9805ad..3f8669b527f 100644 --- a/ethexe/malachite/service/src/config.rs +++ b/ethexe/malachite/service/src/config.rs @@ -10,10 +10,13 @@ use std::{net::SocketAddr, path::PathBuf}; +use ethexe_malachite_core::MalachiteConfigEnvironment; pub use ethexe_malachite_core::{Multiaddr, ValidatorEntry}; #[derive(Clone, Debug)] pub struct MalachiteConfig { + pub env: MalachiteConfigEnvironment, + /// Gas allowance per block. pub gas_allowance: u64, @@ -81,6 +84,7 @@ impl MalachiteConfig { /// (see [`Self::with_validators`]). pub fn from_home_dir(home_dir: PathBuf) -> Self { Self { + env: Default::default(), gas_allowance: Self::DEFAULT_GAS_ALLOWANCE, canonical_quarantine: Self::DEFAULT_CANONICAL_QUARANTINE, post_quarantine_delay: Self::DEFAULT_POST_QUARANTINE_DELAY, diff --git a/ethexe/malachite/service/src/lib.rs b/ethexe/malachite/service/src/lib.rs index 0a80087c01e..8849e41fd95 100644 --- a/ethexe/malachite/service/src/lib.rs +++ b/ethexe/malachite/service/src/lib.rs @@ -61,16 +61,18 @@ pub use crate::{ service::MalachiteService, tx_validity::{MIN_EXECUTABLE_BALANCE_FOR_INJECTED_MESSAGES, TxValidity, TxValidityChecker}, }; -use ethexe_common::injected::PurgedTransaction; pub use ethexe_common::{ injected::SignedInjectedTransaction, malachite::{ProcessQueuesLimits, ProgressTasksLimits, Transaction, Transactions}, }; pub use ethexe_malachite_core::{ - Multiaddr, PeerId, derive_libp2p_secret, libp2p_peer_id as malachite_libp2p_peer_id, + MalachiteConfigEnvironment, Multiaddr, PeerId, derive_libp2p_secret, + libp2p_peer_id as malachite_libp2p_peer_id, }; pub use gprimitives::H256; +use ethexe_common::injected::PurgedTransaction; + /// Ethexe-shaped commit certificate; `block_hash` is the Blake2b envelope hash. #[derive(Clone, Debug, PartialEq, Eq, Default)] pub struct CommitCertificate { diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index bd7b6e6ddbb..eb157c9d763 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -141,6 +141,7 @@ impl MalachiteService { // and travel into the externalities; they never reach // ethexe-malachite-core. let svc_cfg = ethexe_malachite_core::MalachiteConfig { + env: config.env, listen_addr: config.listen_addr, base: config.home_dir.clone(), persistent_peers: config.persistent_peers.clone(), diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index f6512714ccf..24f0b484ff4 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -121,6 +121,7 @@ fn build_config( pub_key: gsigner::schemes::secp256k1::PublicKey, ) -> MalachiteConfig { MalachiteConfig { + env: Default::default(), gas_allowance: MalachiteConfig::DEFAULT_GAS_ALLOWANCE, canonical_quarantine: 0, post_quarantine_delay: 0, diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index af463d9bae0..ef8caca2b8b 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -39,8 +39,9 @@ use ethexe_ethereum::{ router::RouterQuery, }; use ethexe_malachite::{ - InjectedTxMempool, MalachiteConfig, MalachiteService, Multiaddr as MalachiteMultiaddr, PeerId, - ValidatorEntry, derive_libp2p_secret, malachite_libp2p_peer_id, + InjectedTxMempool, MalachiteConfig, MalachiteConfigEnvironment, MalachiteService, + Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, derive_libp2p_secret, + malachite_libp2p_peer_id, }; use ethexe_network::{NetworkConfig, NetworkRuntimeConfig, NetworkService, export::Multiaddr}; use ethexe_observer::{ @@ -1193,6 +1194,7 @@ impl Node { .with_listen_addr(listen_addr) .with_persistent_peers(persistent_peers) .with_validators(validators); + mc.env = MalachiteConfigEnvironment::Test; mc.canonical_quarantine = self.canonical_quarantine; mc.post_quarantine_delay = self.post_quarantine_delay; let mempool = std::sync::Arc::new(InjectedTxMempool::new(self.db.clone())); From 908ed78b60f757566784a6b6052f43a022fa49ec Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Mon, 8 Jun 2026 21:09:02 +0300 Subject: [PATCH 19/72] Start Malachite app in `run_inner` --- ethexe/malachite/core/src/service.rs | 19 ++++++++++++++++++- ethexe/malachite/service/src/service.rs | 6 ++++++ ethexe/service/src/lib.rs | 5 +++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/ethexe/malachite/core/src/service.rs b/ethexe/malachite/core/src/service.rs index d5adb8d104e..a682eeadeb4 100644 --- a/ethexe/malachite/core/src/service.rs +++ b/ethexe/malachite/core/src/service.rs @@ -41,7 +41,10 @@ use std::{ task::{Context as TaskContext, Poll}, time::Duration, }; -use tokio::{sync::mpsc, task::JoinHandle}; +use tokio::{ + sync::{mpsc, oneshot}, + task::JoinHandle, +}; /// Trait-object-friendly facade for the service. The stream carries /// only fatal app-task errors — successful events reach the @@ -50,6 +53,7 @@ pub trait MService: Stream + Send + Unpin {} /// Application-agnostic Malachite BFT consensus service. pub struct MalachiteService> { + start_tx: Option>, errors_rx: mpsc::UnboundedReceiver, engine: EngineHandle, app_handle: JoinHandle<()>, @@ -261,9 +265,12 @@ impl> MalachiteService { )?; // ---- spawn app task ---- + let (start_tx, start_rx) = oneshot::channel(); let (errors_tx, errors_rx) = mpsc::unbounded_channel(); let externalities_for_task = Arc::clone(&externalities); let app_handle = tokio::spawn(async move { + start_rx.await.expect("start sender has been dropped"); + if let Err(e) = app::run::(state, channels, externalities_for_task).await { tracing::error!(target: "ethexe-malachite-core", error = %e, "app task terminated"); let _ = errors_tx.send(e); @@ -271,6 +278,7 @@ impl> MalachiteService { }); Ok(Self { + start_tx: Some(start_tx), errors_rx, engine, app_handle, @@ -281,6 +289,14 @@ impl> MalachiteService { }) } + pub fn start_app_task(&mut self) { + self.start_tx + .take() + .expect("app task has been already started") + .send(()) + .expect("start receiver has been dropped"); + } + /// Swap the active validator set used at the next height start. /// Malachite's `StartHeight` snapshots the set at the height /// start, so the current height runs to completion with whatever @@ -312,6 +328,7 @@ impl> Stream for MalachiteService type Item = anyhow::Error; fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + debug_assert!(self.start_tx.is_none(), "app task is not started"); self.errors_rx.poll_recv(cx) } } diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index eb157c9d763..56684b20e84 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -192,6 +192,12 @@ impl MalachiteService { }) } + pub fn start_app_task(&mut self) { + if let Some(inner) = &mut self.inner { + inner.start_app_task(); + } + } + /// Hand an injected transaction to the mempool. The local /// producer pulls from the same pool when assembling the next MB. /// diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 281bd71770b..09e75fd0193 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -603,11 +603,12 @@ impl Service { fast_sync: _, validator_address, validator_pub_key, - shutdown_rx, + mut shutdown_rx, #[cfg(test)] sender, } = self; - let mut shutdown_rx = shutdown_rx; + + malachite.as_mut().expect("always Some").start_app_task(); let (mut rpc_handle, mut rpc) = if let Some(rpc) = rpc { log::info!("🌐 Rpc server starting at: {}", rpc.port()); From 71eda40872b7d44bbdf336bfe60e7a98aa46fd9c Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Mon, 8 Jun 2026 21:11:30 +0300 Subject: [PATCH 20/72] Manual fixes --- ethexe/service/src/fast_sync.rs | 77 +++++++++++++++------------------ 1 file changed, 34 insertions(+), 43 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index f7af335d160..db2d04245be 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -7,8 +7,7 @@ use crate::Service; use alloy::eips::BlockId; use anyhow::{Context, Result, ensure}; use ethexe_common::{ - Address, BlockData, CodeAndIdUnchecked, Digest, ProgramStates, SimpleBlockData, - StateHashWithQueueSize, + Address, BlockData, CodeAndIdUnchecked, Digest, ProgramStates, StateHashWithQueueSize, db::{ BlockMetaStorageRO, CodesStorageRO, CodesStorageRW, CompactMb, ConfigStorageRO, GlobalsStorageRW, MbStorageRW, OnChainStorageRW, PreparedBlockData, @@ -658,15 +657,20 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { } = event_data; let latest_committed_eb = latest_committed_eb.unwrap_or(genesis_block_hash); - let BlockData { - hash: block_hash, - header, - events, - } = block_loader.load(latest_committed_eb, None).await?; + let block_data = block_loader.load(latest_committed_eb, None).await?; - let code_ids = collect_code_ids(observer, genesis_block.header.height, header.height).await?; - let program_code_ids = - collect_program_code_ids(observer, genesis_block.header.height, header.height).await?; + let code_ids = collect_code_ids( + observer, + genesis_block.header.height, + block_data.header.height, + ) + .await?; + let program_code_ids = collect_program_code_ids( + observer, + genesis_block.header.height, + block_data.header.height, + ) + .await?; let program_state_hashes = collect_program_state_hashes(observer, state_query_block, &program_code_ids).await?; @@ -674,23 +678,25 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { instrument_codes(compute, db, code_ids).await?; - let schedule = ScheduleRestorer::from_storage(db, &program_states, header.height)?.restore(); + let schedule = + ScheduleRestorer::from_storage(db, &program_states, block_data.header.height)?.restore(); - set_tx_pool_data_requirement(db, &block_loader, header.height).await?; + set_tx_pool_data_requirement(db, &block_loader, block_data.header.height).await?; for (program_id, code_id) in program_code_ids { db.set_program_code_id(program_id, code_id); } let latest_era_with_committed_validators = - latest_era_with_committed_validators(db, &observer.router_query(), block_hash).await?; + latest_era_with_committed_validators(db, &observer.router_query(), latest_committed_eb) + .await?; ethexe_common::setup_block_in_db( db, - block_hash, + latest_committed_eb, PreparedBlockData { - header, - events, + header: block_data.header, + events: block_data.events.clone(), latest_era_with_committed_validators, // NOTE: there is no invariant that fast sync should recover codes queue codes_queue: Default::default(), @@ -707,17 +713,8 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { meta.last_advanced_eb = latest_committed_eb; }); - let transactions_hash = db.set_transactions(Transactions::default()); - // The committed MB is a fast-sync boundary: chain data gives us the head - // hash, but not the Malachite envelope `(parent, height, payload_hash)`. - // Store a synthetic compact row so DB walks have an anchor without - // pretending to restore the pre-boundary MB chain. - let compact = CompactMb { - parent: H256::zero(), - height: 0, - transactions_hash, - }; - db.set_mb_compact_block(latest_committed_mb, compact); + // NOTE: Malachite should restore `CompactMb` by itself + //db.set_mb_compact_block(latest_committed_mb, compact); db.set_mb_program_states(latest_committed_mb, program_states.clone()); db.set_mb_schedule(latest_committed_mb, schedule); @@ -727,33 +724,27 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { db.set_mb_outcome(latest_committed_mb, Default::default()); db.globals_mutate(|globals| { - globals.start_block_hash = block_hash; - globals.latest_synced_eb = SimpleBlockData { - hash: block_hash, - header, - }; - globals.latest_prepared_eb_hash = block_hash; + globals.start_block_hash = latest_committed_eb; + globals.latest_synced_eb = block_data.to_simple(); + globals.latest_prepared_eb_hash = latest_committed_eb; globals.latest_finalized_mb_hash = latest_committed_mb; globals.latest_computed_mb_hash = latest_committed_mb; }); - if !block_hash.is_zero() - && let Some(malachite) = malachite.as_mut() - { - malachite.receive_new_chain_head(SimpleBlockData { - hash: block_hash, - header, - }); + if let Some(malachite) = malachite.as_mut() { + malachite.receive_new_chain_head(block_data.to_simple()); } log::info!( - "Fast synchronization done: synced to {block_hash:?}, height {:?}, MB {latest_committed_mb}", - header.height + "Fast synchronization done: synced to {latest_committed_eb:?}, height {height:?}, MB {latest_committed_mb}", + height = block_data.header.height ); #[cfg(test)] sender - .send(crate::tests::utils::TestingEvent::FastSyncDone(block_hash)) + .send(crate::tests::utils::TestingEvent::FastSyncDone( + latest_committed_eb, + )) .await; Ok(()) From 19995415bef6c62f935183f798895501b8e0f19b Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Mon, 8 Jun 2026 21:12:13 +0300 Subject: [PATCH 21/72] Change some `TestingEventReceiver` methods --- ethexe/service/src/tests/utils/events.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ethexe/service/src/tests/utils/events.rs b/ethexe/service/src/tests/utils/events.rs index 849a4978344..d1137d38f54 100644 --- a/ethexe/service/src/tests/utils/events.rs +++ b/ethexe/service/src/tests/utils/events.rs @@ -309,7 +309,6 @@ impl KickingStream> { } impl TestingEventReceiver { - #[allow(dead_code)] pub async fn find_block_synced(&mut self) -> H256 { self.find_map(|event| { if let TestingEvent::Observer(ObserverEvent::BlockSynced(block_hash)) = event { @@ -323,7 +322,6 @@ impl TestingEventReceiver { /// Drive the compute stream forward until a `BlockPrepared(target)` event /// arrives. - #[allow(dead_code)] pub async fn find_block_prepared(&mut self, target: H256) -> H256 { self.find_map(|event| match event { TestingEvent::Compute(ComputeEvent::BlockPrepared(h)) if h == target => Some(h), @@ -333,10 +331,13 @@ impl TestingEventReceiver { } /// Wait until any MB becomes computed, returning its hash. - #[allow(dead_code)] - pub async fn find_any_mb_computed(&mut self) -> H256 { + pub async fn find_mb_computed(&mut self, mb_hash: H256) -> H256 { self.find_map(|event| match event { - TestingEvent::Compute(ComputeEvent::MbComputed(mb_hash)) => Some(mb_hash), + TestingEvent::Compute(ComputeEvent::MbComputed(event_mb_hash)) + if event_mb_hash == mb_hash => + { + Some(mb_hash) + } _ => None, }) .await @@ -348,7 +349,7 @@ impl TestingEventReceiver { /// ancestor of this MB's `last_advanced_eb` (i.e., it sits inside /// the eth-chain segment this MB advanced over). #[allow(dead_code)] - pub async fn wait_till_eth_block_finalized_in_mb(&mut self, target_eth_block: H256) { + pub async fn find_advanced_mb(&mut self, eb_hash: H256) -> H256 { self.find_map_with_db(|db, event| { let TestingEvent::Malachite(MalachiteEvent::BlockFinalized { mb_hash, .. }) = event else { @@ -368,8 +369,8 @@ impl TestingEventReceiver { // covers it. let mut cursor = last_advanced; while cursor != prev_advanced { - if cursor == target_eth_block { - return Some(()); + if cursor == eb_hash { + return Some(mb_hash); } let header = db.block_header(cursor)?; if header.parent_hash.is_zero() { From 74b4c6fef2379aaf2193475c31d2f789e4d9e579 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Mon, 8 Jun 2026 21:12:37 +0300 Subject: [PATCH 22/72] Initial manual fast sync test (`fast_sync2`) --- Cargo.lock | 1 + ethexe/service/Cargo.toml | 1 + ethexe/service/src/tests/mod.rs | 354 +++++++++++++++++++++++++++++++- 3 files changed, 352 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ebdd917e949..685e507fdaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6053,6 +6053,7 @@ dependencies = [ "demo-async-init", "demo-delayed-sender-ethexe", "demo-fungible-token", + "demo-mul-by-const", "demo-piggy-bank", "demo-ping", "demo-reply-callback", diff --git a/ethexe/service/Cargo.toml b/ethexe/service/Cargo.toml index 00eb0b3c584..24509cc3508 100644 --- a/ethexe/service/Cargo.toml +++ b/ethexe/service/Cargo.toml @@ -80,6 +80,7 @@ demo-async-init = { workspace = true, features = ["debug", "ethexe"] } demo-piggy-bank = { workspace = true, features = ["debug", "ethexe"] } demo-reply-callback = { workspace = true, features = ["debug", "ethexe"] } demo-fungible-token = { workspace = true } +demo-mul-by-const = { workspace = true } demo-delayed-sender-ethexe = { workspace = true, features = [ "debug", "ethexe", diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 0e6755b88af..a947cd4095e 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -1479,9 +1479,7 @@ async fn reorg_deeper_than_quarantine() { let latest_block = env.latest_block().await; test_info!("Waiting for {latest_block} to be finalized in MB"); - node.events() - .wait_till_eth_block_finalized_in_mb(latest_block.hash) - .await; + node.events().find_advanced_mb(latest_block.hash).await; test_info!("📗 Reverting Anvil to deep snapshot — past quarantine"); env.provider @@ -1499,7 +1497,7 @@ async fn reorg_deeper_than_quarantine() { let mut receiver = node.new_events(); // Here we take in account kicking stream - latest_block is not passed quarantine yet, // but kicks will generate new anvil blocks, but still this block cannot be finalized in mb, because branch is broken. - let waiting_future = receiver.wait_till_eth_block_finalized_in_mb(latest_block.hash); + let waiting_future = receiver.find_advanced_mb(latest_block.hash); tokio::time::timeout(Duration::from_secs(20), waiting_future) .await .expect_err("block should not be finalized within 20 seconds after deep reorg"); @@ -3165,6 +3163,354 @@ async fn reply_callback() { stop_nodes([node]).await; } +#[tokio::test] +#[ntest::timeout(120_000)] +async fn fast_sync2() { + init_logger(); + + // let assert_chain = |latest_block, fast_synced_block, alice: &Node, bob: &Node| { + // log::info!("Assert chain in range {latest_block}..{fast_synced_block}"); + // + // IntegrityVerifier::new(alice.db.clone()) + // .verify_chain(latest_block, fast_synced_block) + // .expect("failed to verify Alice database"); + // + // IntegrityVerifier::new(bob.db.clone()) + // .verify_chain(latest_block, fast_synced_block) + // .expect("failed to verify Bob database"); + // + // let alice_globals = alice.db.globals(); + // let bob_globals = bob.db.globals(); + // assert_eq!( + // alice_globals.latest_computed_announce_hash, + // bob_globals.latest_computed_announce_hash + // ); + // assert_eq!( + // alice_globals.latest_prepared_block_hash, + // bob_globals.latest_prepared_block_hash + // ); + // + // let mut block = latest_block; + // loop { + // if fast_synced_block == block { + // break; + // } + // + // log::trace!("assert block {block}"); + // + // // Check block meta, exclude codes_queue and announces, which can vary, and it's ok + // let alice_meta = alice.db.block_meta(block); + // let bob_meta = bob.db.block_meta(block); + // assert!( + // alice_meta.prepared && bob_meta.prepared, + // "Block {block} is not prepared for alice or bob" + // ); + // assert_eq!( + // alice_meta.last_committed_announce, + // bob_meta.last_committed_announce + // ); + // assert_eq!( + // alice_meta.last_committed_batch, + // bob_meta.last_committed_batch + // ); + // + // let alice_announces = alice.db.block_announces(block); + // let bob_announces = bob.db.block_announces(block); + // let Some((alice_announces, bob_announces)) = alice_announces.zip(bob_announces) else { + // panic!("alice or bob has no announces"); + // }; + // + // for &announce_hash in alice_announces.intersection(&bob_announces) { + // if alice.db.announce_meta(announce_hash).computed + // != bob.db.announce_meta(announce_hash).computed + // { + // continue; + // } + // + // assert_eq!( + // alice.db.announce_program_states(announce_hash), + // bob.db.announce_program_states(announce_hash) + // ); + // assert_eq!( + // alice.db.announce_outcome(announce_hash), + // bob.db.announce_outcome(announce_hash) + // ); + // assert_eq!( + // alice.db.announce_outcome(announce_hash), + // bob.db.announce_outcome(announce_hash) + // ); + // } + // + // assert_eq!(alice.db.block_header(block), bob.db.block_header(block)); + // assert_eq!(alice.db.block_events(block), bob.db.block_events(block)); + // assert_eq!(alice.db.block_synced(block), bob.db.block_synced(block)); + // + // let header = alice.db.block_header(block).unwrap(); + // block = header.parent_hash; + // } + // }; + + let assert_chain = |latest_block, fast_synced_block, alice: &Node, bob: &Node| { + log::info!("Assert chain in range {latest_block}..{fast_synced_block}"); + + IntegrityVerifier::new(alice.db.clone()) + .verify_chain(latest_block, fast_synced_block) + .expect("failed to verify Alice database"); + + IntegrityVerifier::new(bob.db.clone()) + .verify_chain(latest_block, fast_synced_block) + .expect("failed to verify Bob database"); + + let mut eb_hash = latest_block; + loop { + if fast_synced_block == eb_hash { + break; + } + + log::trace!("assert EB {eb_hash}"); + + // Check block meta, exclude codes_queue and announces, which can vary, and it's ok + let alice_meta = alice.db.block_meta(eb_hash); + let bob_meta = bob.db.block_meta(eb_hash); + assert_eq!(alice_meta, bob_meta); + // assert!( + // alice_meta.prepared && bob_meta.prepared, + // "Block {block} is not prepared for alice or bob" + // ); + // assert_eq!( + // alice_meta.last_committed_announce, + // bob_meta.last_committed_announce + // ); + // assert_eq!( + // alice_meta.last_committed_batch, + // bob_meta.last_committed_batch + // ); + + assert_eq!(alice.db.block_header(eb_hash), bob.db.block_header(eb_hash)); + assert_eq!(alice.db.block_events(eb_hash), bob.db.block_events(eb_hash)); + assert_eq!(alice.db.block_synced(eb_hash), bob.db.block_synced(eb_hash)); + + let mb_hash = bob_meta.last_committed_mb.unwrap(); + + let alice_compact_block = alice.db.mb_compact_block(mb_hash); + let bob_compact_block = bob.db.mb_compact_block(mb_hash); + assert_eq!(alice_compact_block, bob_compact_block); + + if let (Some(alice_compact_block), Some(bob_compact_block)) = + (alice_compact_block, bob_compact_block) + { + let alice_transactions = + alice.db.transactions(alice_compact_block.transactions_hash); + let bob_transactions = bob.db.transactions(bob_compact_block.transactions_hash); + assert_eq!(alice_transactions, bob_transactions); + } + + let alice_program_states = alice.db.mb_program_states(mb_hash); + let bob_program_states = bob.db.mb_program_states(mb_hash); + assert_eq!(alice_program_states, bob_program_states); + + let alice_outcome = alice.db.mb_outcome(mb_hash); + let bob_outcome = bob.db.mb_outcome(mb_hash); + assert_eq!(alice_outcome, bob_outcome); + + let alice_schedule = alice.db.mb_schedule(mb_hash); + let bob_schedule = bob.db.mb_schedule(mb_hash); + assert_eq!(alice_schedule, bob_schedule); + + let alice_meta = alice.db.mb_meta(mb_hash); + let bob_meta = bob.db.mb_meta(mb_hash); + assert_eq!(alice_meta, bob_meta); + + let header = alice.db.block_header(eb_hash).unwrap(); + eb_hash = header.parent_hash; + } + + // let mut block_hash = latest_block; + // loop { + // if fast_synced_block == block_hash { + // break; + // } + // + // log::trace!("assert MB {mb_hash}"); + // + // // Check block meta, exclude codes_queue and announces, which can vary, and it's ok + // + // let alice_compact_block = alice.db.mb_compact_block(block_hash); + // let bob_compact_block = bob.db.mb_compact_block(block_hash); + // assert_eq!(alice_compact_block, bob_compact_block); + // + // if let (Some(alice_compact_block), Some(bob_compact_block)) = + // (alice_compact_block, bob_compact_block) + // { + // let alice_transactions = + // alice.db.transactions(alice_compact_block.transactions_hash); + // let bob_transactions = alice.db.transactions(bob_compact_block.transactions_hash); + // assert_eq!(alice_transactions, bob_transactions); + // } + // + // let alice_program_states = alice.db.mb_program_states(block_hash); + // let bob_program_states = bob.db.mb_program_states(block_hash); + // assert_eq!(alice_program_states, bob_program_states); + // + // let alice_outcome = alice.db.mb_outcome(block_hash); + // let bob_outcome = alice.db.mb_outcome(block_hash); + // assert_eq!(alice_outcome, bob_outcome); + // + // let alice_schedule = alice.db.mb_schedule(block_hash); + // let bob_schedule = bob.db.mb_schedule(block_hash); + // assert_eq!(alice_schedule, bob_schedule); + // + // let alice_meta = alice.db.mb_meta(block_hash); + // let bob_meta = bob.db.mb_meta(block_hash); + // assert_eq!(alice_meta, bob_meta); + // + // block_hash = alice_compact_block.unwrap().parent; + // } + }; + + let config = TestEnvConfig { + network: EnvNetworkConfig::Enabled, + ..Default::default() + }; + let mut env = TestEnv::new(config).await.unwrap(); + + test_info!("📗 Starting Alice"); + let mut alice = env + .new_node(NodeConfig::named("Alice").validator(env.validators[0])) + .await; + alice.start_service().await; + + test_info!("📗 Creating `demo-mul-by-const` programs"); + + let code_info = env + .upload_code(demo_mul_by_const::WASM_BINARY) + .await + .unwrap() + .wait_for() + .await + .unwrap(); + + let code_id = code_info.code_id; + let mut program_ids = [ActorId::zero(); 8]; + + for (i, program_id) in program_ids.iter_mut().enumerate() { + let program_info = env + .create_program_with_params(code_id, H256([i as u8; 32]), None, 500_000_000_000_000) + .await + .unwrap() + .wait_for() + .await + .unwrap(); + + *program_id = program_info.program_id; + + let value = i as u64 % 3; + let _reply_info = env + .send_message(program_info.program_id, &value.encode()) + .await + .unwrap() + .wait_for() + .await + .unwrap(); + } + + let latest_block = env.latest_block().await.hash; + alice.events().find_advanced_mb(latest_block).await; + + test_info!("Starting Bob (fast-sync)"); + let mut bob = env.new_node(NodeConfig::named("Bob").fast_sync()).await; + + bob.start_service().await; + + test_info!("📗 Sending messages to programs"); + + for (i, program_id) in program_ids.into_iter().enumerate() { + let reply_info = env + .send_message(program_id, &(i as u64).encode()) + .await + .unwrap() + .wait_for() + .await + .unwrap(); + assert_eq!( + reply_info.code, + ReplyCode::Success(SuccessReplyReason::Manual) + ); + } + + let latest_block = env.latest_block().await.hash; + // alice + // .events() + // .find_mb_computed() + // .find_announce_computed(latest_block) + // .await; + let mut alice_events = alice.events(); + let mut bob_events = bob.events(); + let (alice_mb_hash, bob_mb_hash) = tokio::join!( + alice_events.find_advanced_mb(latest_block), + bob_events.find_advanced_mb(latest_block) + ); + tokio::join!( + alice_events.find_mb_computed(alice_mb_hash), + bob_events.find_mb_computed(bob_mb_hash) + ); + + test_info!("📗 Stopping Bob"); + bob.stop_service().await; + + assert_chain( + latest_block, + bob.latest_fast_synced_block.take().unwrap(), + &alice, + &bob, + ); + + for (i, program_id) in program_ids.into_iter().enumerate() { + let i = (i * 3) as u64; + let reply_info = env + .send_message(program_id, &i.encode()) + .await + .unwrap() + .wait_for() + .await + .unwrap(); + assert_eq!( + reply_info.code, + ReplyCode::Success(SuccessReplyReason::Manual) + ); + } + + env.skip_blocks(100).await; + + let latest_block = env.latest_block().await.hash; + alice.events().find_advanced_mb(latest_block).await; + + test_info!("📗 Starting Bob again to check how it handles partially empty database"); + bob.start_service().await; + + // Mine some blocks so Bob can produce the event we will wait for. + // We mine several blocks here to ensure that Bob and Alice would converge to the same chain of announces. + // Why do we need that? Because Bob was disabled he missed some announces that Alice produced, + // this announces was not committed, so Bob would not see them during fast-sync + // and would not have them in his database. This is normal situation, after a few blocks Bob and Alice should + // converge to the same chain of announces. + for _ in 0..env.commitment_delay_limit.get() { + env.skip_blocks(1).await; + } + + let latest_block = env.latest_block().await.hash; + let alice_latest_advanced_mb = alice.events().find_advanced_mb(latest_block).await; + let bob_latest_advanced_mb = bob.events().find_advanced_mb(latest_block).await; + assert_eq!(alice_latest_advanced_mb, bob_latest_advanced_mb); + + assert_chain( + latest_block, + bob.latest_fast_synced_block.take().unwrap(), + &alice, + &bob, + ); +} + #[tokio::test] #[ntest::timeout(180_000)] async fn fast_sync() { From ab6f14fee7c7c113eace62e31aaffa82aaf620b8 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Mon, 8 Jun 2026 23:45:25 +0300 Subject: [PATCH 23/72] Suppress pre-boundary ValueSync replay events --- ethexe/malachite/core/src/app.rs | 209 +++++- ethexe/malachite/core/src/externalities.rs | 38 +- ethexe/malachite/core/src/lib.rs | 2 +- ethexe/malachite/core/src/store.rs | 32 +- .../malachite/core/tests/multi_validators.rs | 14 +- ethexe/malachite/service/src/externalities.rs | 632 +++++++++++++++++- ethexe/malachite/service/src/service.rs | 12 +- ethexe/service/src/fast_sync.rs | 15 +- ethexe/service/src/lib.rs | 6 +- 9 files changed, 914 insertions(+), 46 deletions(-) diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index f9ca209a56b..f8342fc6069 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -31,7 +31,7 @@ use crate::{ codec::{decode_value, encode_value}, context::{Height, MalachiteCtx, ProposalPart, ValueId}, - externalities::{BlockPayload, Externalities}, + externalities::{BlockPayload, CallbackOrigin, Externalities}, state::State, store::BlockEntry, streaming::ProposalParts, @@ -352,7 +352,8 @@ where self.state.store.store_undecided_proposal(&proposed)?; let block = Block::

::decode(&mut &proposed.value.block_bytes[..]) .map_err(|e| anyhow!("decoding Block from pending proposal: {e}"))?; - self.record_assembled_block(block).await + self.record_assembled_block(block, CallbackOrigin::Live) + .await }; if let Err(e) = promote.await { error!(?e, "rejecting invalid pending proposal"); @@ -406,7 +407,8 @@ where // Hook process_mb_proposal at proposal-assembly time on the // proposer side. cascade_save guarantees ancestor-first // ordering against the application. - self.record_assembled_block(block).await?; + self.record_assembled_block(block, CallbackOrigin::Live) + .await?; Ok(locally) } @@ -454,7 +456,8 @@ where self.state.store.store_undecided_proposal(&proposed)?; let block = Block::

::decode(&mut &proposed.value.block_bytes[..]) .map_err(|e| anyhow!("decoding Block from received proposal: {e}"))?; - self.record_assembled_block(block).await?; + self.record_assembled_block(block, CallbackOrigin::Live) + .await?; Ok(Some(proposed)) } } @@ -477,7 +480,7 @@ where .state .commit(certificate.clone()) .map_err(FinalizationError::NonFatal)?; - self.ingest_finalized(certificate, block_bytes) + self.ingest_finalized(certificate, block_bytes, CallbackOrigin::Live) .await .context("ingest finalized") .map_err(FinalizationError::Fatal) @@ -507,7 +510,8 @@ where // missing ancestor anyway — see `Store::save_chain`). let block = Block::

::decode(&mut &proposed.value.block_bytes[..]) .map_err(|e| anyhow!("decoding Block from synced value: {e}"))?; - self.record_assembled_block(block).await?; + self.record_assembled_block(block, CallbackOrigin::ValueSyncReplay) + .await?; } Ok(parsed) } @@ -641,8 +645,9 @@ where /// dedup is idempotent and `cascade_save` skips already-saved /// entries, so the application's `process_mb_proposal` runs at /// most once per `block_hash`. - async fn record_assembled_block(&self, block: Block

) -> Result<()> { + async fn record_assembled_block(&self, block: Block

, origin: CallbackOrigin) -> Result<()> { let block_hash = block.hash(); + self.state.store.set_mb_origin_once(block_hash, origin)?; self.state.store.insert_block(BlockEntry::

{ block_hash, parent_hash: block.parent_hash, @@ -657,7 +662,12 @@ where .store .cascade_save(vec![block_hash], |hash, blk| { let ext = Arc::clone(&self.externalities); - async move { ext.process_mb_proposal(hash, blk).await } + let store = self.state.store.clone(); + let fallback = origin; + async move { + let cb_origin = store.mb_origin(hash)?.unwrap_or(fallback); + ext.process_mb_proposal(hash, blk, cb_origin).await + } }) .await } @@ -673,7 +683,12 @@ where /// [`Store::cascade_finalize`] silently no-ops on an unsaved /// ancestor (the `finalize_chain` walk returns `None`), and the /// `errors_tx` channel surfaces the contract breach upstream. - async fn ingest_finalized(&self, cert: EngineCert, block_bytes: Vec) -> Result<()> { + async fn ingest_finalized( + &self, + cert: EngineCert, + block_bytes: Vec, + fallback_origin: CallbackOrigin, + ) -> Result<()> { let block = Block::

::decode(&mut &block_bytes[..]) .map_err(|e| anyhow!("decoding Block at finalize: {e}"))?; let block_hash = block.hash(); @@ -721,7 +736,11 @@ where .store .cascade_finalize(vec![block_hash], |hash, cert| { let ext = Arc::clone(&self.externalities); - async move { ext.process_mb_finalized(hash, cert).await } + let store = self.state.store.clone(); + async move { + let cb_origin = store.mb_origin(hash)?.unwrap_or(fallback_origin); + ext.process_mb_finalized(hash, cert, cb_origin).await + } }) .await?; Ok(()) @@ -767,6 +786,7 @@ fn compute_value_id_from_parts(parts: &ProposalParts) -> ValueId { mod tests { use super::*; use crate::{ + codec::encode_value, context::{ProposalData, ProposalInit, Validator, ValidatorSet, Value}, signing::{MalachiteSigner, libp2p_peer_id, private_key_from_bytes}, state::SharedValidatorSet, @@ -777,7 +797,11 @@ mod tests { ConsensusRequest, NetworkRequest, app::{events::TxEvent, streaming::StreamId}, }; - use std::time::Duration; + use malachitebft_core_types::Value as MalachiteBftValue; + use std::{ + sync::{Arc, Mutex}, + time::Duration, + }; use tempfile::TempDir; use tokio::sync::mpsc; @@ -788,10 +812,20 @@ mod tests { #[async_trait] impl Externalities for NoopExt { - async fn process_mb_proposal(&self, _: H256, _: Block) -> Result<()> { + async fn process_mb_proposal( + &self, + _: H256, + _: Block, + _: CallbackOrigin, + ) -> Result<()> { Ok(()) } - async fn process_mb_finalized(&self, _: H256, _: CommitCertificate) -> Result<()> { + async fn process_mb_finalized( + &self, + _: H256, + _: CommitCertificate, + _: CallbackOrigin, + ) -> Result<()> { Ok(()) } async fn build_block_above(&self, _: H256) -> Result { @@ -802,6 +836,65 @@ mod tests { } } + #[derive(Clone, Default)] + struct OriginRecordingExt { + proposal_origins: Arc>>, + finalized_origins: Arc>>, + } + + impl OriginRecordingExt { + fn proposal_origins(&self) -> Vec { + self.proposal_origins + .lock() + .expect("proposal_origins poisoned") + .clone() + } + + fn finalized_origins(&self) -> Vec { + self.finalized_origins + .lock() + .expect("finalized_origins poisoned") + .clone() + } + } + + #[async_trait] + impl Externalities for OriginRecordingExt { + async fn process_mb_proposal( + &self, + _: H256, + _: Block, + origin: CallbackOrigin, + ) -> Result<()> { + self.proposal_origins + .lock() + .expect("proposal_origins poisoned") + .push(origin); + Ok(()) + } + + async fn process_mb_finalized( + &self, + _: H256, + _: CommitCertificate, + origin: CallbackOrigin, + ) -> Result<()> { + self.finalized_origins + .lock() + .expect("finalized_origins poisoned") + .push(origin); + Ok(()) + } + + async fn build_block_above(&self, _: H256) -> Result { + Ok(TestPayload) + } + + async fn validate_block_above(&self, _: H256, _: TestPayload) -> Result { + Ok(true) + } + } + fn test_signer(byte: u8) -> MalachiteSigner { let mut k = [0u8; 32]; k[31] = byte; @@ -844,6 +937,76 @@ mod tests { ] } + /// Sync-path callbacks must be tagged as + /// `CallbackOrigin::ValueSyncReplay` all the way down to + /// application callbacks. + #[tokio::test] + async fn process_synced_value_forwards_value_sync_replay_origin() { + let ext = OriginRecordingExt::default(); + let handler_ext = ext.clone(); + let (mut handler, _dir, proposer) = make_handler_with(1, handler_ext); + let block = Block::::new(H256::zero(), 1, TestPayload); + let block_bytes = block.encode(); + let value_bytes = encode_value(&crate::context::Value::new(block_bytes.clone())); + + let parsed = handler + .process_synced_value(Height::new(1), Round::new(0), proposer, value_bytes) + .await + .expect("process_synced_value should decode and persist the block"); + assert!(parsed.is_some()); + + assert_eq!( + ext.proposal_origins(), + vec![CallbackOrigin::ValueSyncReplay] + ); + + let value_id = crate::context::Value::new(block_bytes).id(); + let cert = malachitebft_core_types::CommitCertificate { + height: Height::new(1), + round: Round::new(0), + value_id, + commit_signatures: Vec::new(), + }; + assert!( + handler.process_finalized(cert).await.is_ok(), + "synced block must be finalized" + ); + assert_eq!( + ext.finalized_origins(), + vec![CallbackOrigin::ValueSyncReplay] + ); + } + + /// Peer/engine proposal callbacks are marked as live traffic and + /// are not mistaken for sync replay callbacks. + #[tokio::test] + async fn process_received_proposal_part_marks_live_origin() { + let ext = OriginRecordingExt::default(); + let handler_ext = ext.clone(); + let (mut handler, _dir, address) = make_handler_with(1, handler_ext); + let block = Block::::new(H256::zero(), 1, TestPayload); + let stream = complete_stream(address, 1, &block.encode()); + + run_stream(&mut handler, test_peer(2), stream) + .await + .expect("process_received_proposal_part should assemble a complete proposal"); + + assert_eq!(ext.proposal_origins(), vec![CallbackOrigin::Live]); + + let value_id = crate::context::Value::new(block.encode()).id(); + let cert = malachitebft_core_types::CommitCertificate { + height: Height::new(1), + round: Round::new(0), + value_id, + commit_signatures: Vec::new(), + }; + assert!( + handler.process_finalized(cert).await.is_ok(), + "proposal should finalize" + ); + assert_eq!(ext.finalized_origins(), vec![CallbackOrigin::Live]); + } + /// Build an [`AppMsgHandler`] with the given `current_height` and /// a fresh on-disk store. Channels are dummy senders/receivers /// — `process_received_proposal_part` never touches them on the @@ -893,8 +1056,8 @@ mod tests { /// `process_received_proposal_part`, returning the final reply /// value (`Some(proposed)` on the same-height happy path, `None` /// when the parts were dropped or buffered). - async fn run_stream( - handler: &mut AppMsgHandler, + async fn run_stream>( + handler: &mut AppMsgHandler, peer: PeerId, stream: Vec>, ) -> Option> { @@ -973,10 +1136,20 @@ mod tests { #[async_trait] impl Externalities for FailingFinalizeExt { - async fn process_mb_proposal(&self, _: H256, _: Block) -> Result<()> { + async fn process_mb_proposal( + &self, + _: H256, + _: Block, + _: CallbackOrigin, + ) -> Result<()> { Ok(()) } - async fn process_mb_finalized(&self, _: H256, _: CommitCertificate) -> Result<()> { + async fn process_mb_finalized( + &self, + _: H256, + _: CommitCertificate, + _: CallbackOrigin, + ) -> Result<()> { Err(anyhow!("application: finalize-side store write failed")) } async fn build_block_above(&self, _: H256) -> Result { @@ -1069,7 +1242,7 @@ mod tests { // `all_ancestors_saved` returns false and the debug-build // assertion fires before the cascade). handler - .record_assembled_block(block) + .record_assembled_block(block, CallbackOrigin::Live) .await .expect("record_assembled_block must succeed under NoopFinalize proposal-side"); diff --git a/ethexe/malachite/core/src/externalities.rs b/ethexe/malachite/core/src/externalities.rs index 14ee6401dab..43da48051cd 100644 --- a/ethexe/malachite/core/src/externalities.rs +++ b/ethexe/malachite/core/src/externalities.rs @@ -6,6 +6,7 @@ use anyhow::Result; use async_trait::async_trait; use parity_scale_codec::{Decode, Encode}; +use serde::{Deserialize, Serialize}; use crate::types::{Block, CommitCertificate, H256}; @@ -20,6 +21,19 @@ pub trait BlockPayload: Clone + Encode + Decode + Send + Sync + 'static {} impl BlockPayload for T where T: Clone + Encode + Decode + Send + Sync + 'static {} +/// Indicates where a proposed/finalized MB callback originated. +/// `CallbackOrigin::ValueSyncReplay` identifies callbacks produced by +/// Malachite replay (ValueSync decided-values). It is *provenance only*: +/// applications must decide how to act on this origin, and the default +/// execution path is to treat it like a normal callback unless a higher +/// layer explicitly opts into replay filtering. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Decode, Encode, Deserialize, Serialize)] +pub enum CallbackOrigin { + #[default] + Live, + ValueSyncReplay, +} + /// Application-side callbacks the consensus service requires. /// /// The service guarantees a strict happens-before ordering for the @@ -60,14 +74,34 @@ pub trait Externalities: Send + Sync + 'static { /// per `mb_hash` over the lifetime of an application instance, /// at proposal-assembly time, after every ancestor's /// `process_mb_proposal` has already returned `Ok`. - async fn process_mb_proposal(&self, mb_hash: H256, block: Block

) -> Result<()>; + /// `origin` indicates whether the callback came from local/live + /// operation (`Live`) or Malachite replay (`ValueSyncReplay`). + /// `ValueSyncReplay` is provenance only; downstream implementations + /// may apply additional policy (for example, replay suppression when + /// an execution snapshot boundary is explicitly enabled). + async fn process_mb_proposal( + &self, + mb_hash: H256, + block: Block

, + origin: CallbackOrigin, + ) -> Result<()>; /// Mark `mb_hash` as finalized and durable. /// /// `cert` is the BFT commit certificate for the height of /// `mb_hash`. The application typically forwards `cert` to /// downstream layers (on-chain commits, light clients, etc.). - async fn process_mb_finalized(&self, mb_hash: H256, cert: CommitCertificate) -> Result<()>; + /// `origin` indicates whether the callback came from local/live + /// operation (`Live`) or Malachite replay (`ValueSyncReplay`). + /// `ValueSyncReplay` is provenance only; downstream implementations + /// may apply additional policy (for example, replay suppression when + /// an execution snapshot boundary is explicitly enabled). + async fn process_mb_finalized( + &self, + mb_hash: H256, + cert: CommitCertificate, + origin: CallbackOrigin, + ) -> Result<()>; /// Build a fresh block payload whose parent has hash /// `parent_mb_hash`. Called only when this node has been elected diff --git a/ethexe/malachite/core/src/lib.rs b/ethexe/malachite/core/src/lib.rs index 4749188e2e4..cf9f04de555 100644 --- a/ethexe/malachite/core/src/lib.rs +++ b/ethexe/malachite/core/src/lib.rs @@ -82,7 +82,7 @@ pub use crate::{ Environment as MalachiteConfigEnvironment, MalachiteConfig, Multiaddr, NodeRole, ValidatorEntry, }, - externalities::{BlockPayload, Externalities}, + externalities::{BlockPayload, CallbackOrigin, Externalities}, service::{MService, MalachiteService}, signing::{ MalachiteSigner, PrivateKey, PublicKey, Signature, derive_libp2p_secret, diff --git a/ethexe/malachite/core/src/store.rs b/ethexe/malachite/core/src/store.rs index 7101678d3ec..09dca5a38d0 100644 --- a/ethexe/malachite/core/src/store.rs +++ b/ethexe/malachite/core/src/store.rs @@ -16,6 +16,7 @@ //! - `0x05` `(height,round,value_id)` → engine undecided proposal //! - `0x06` `(height,round,value_id)` → buffered proposal parts //! - `0x07` `height_be[8]` → engine-side `CommitCertificate` +//! - `0x08` `mb_hash[32]` → callback origin for per-block processing //! //! Children of the genesis (parent_hash == [`H256::zero`]) live under //! the bare-zero parent key — same shape as any other parent. @@ -37,7 +38,7 @@ use rocksdb::{DB, Options, WriteBatch}; use crate::{ context::Height, - externalities::BlockPayload, + externalities::{BlockPayload, CallbackOrigin}, types::{Block, CommitCertificate, H256}, }; @@ -49,6 +50,7 @@ mod prefix { pub const UNDECIDED: u8 = 0x05; pub const PENDING_PARTS: u8 = 0x06; pub const ENGINE_CERT: u8 = 0x07; + pub const MB_ORIGIN: u8 = 0x08; } const META_LATEST_FINALIZED: &[u8] = b"latest_finalized"; @@ -143,6 +145,13 @@ impl Store

{ k } + fn key_mb_origin(hash: H256) -> [u8; 33] { + let mut k = [0u8; 33]; + k[0] = prefix::MB_ORIGIN; + k[1..33].copy_from_slice(hash.as_bytes()); + k + } + fn decode_one(bytes: &[u8], what: &'static str) -> Result { T::decode(&mut &bytes[..]).with_context(|| format!("decoding {what}")) } @@ -199,6 +208,27 @@ impl Store

{ } } + /// Persist callback origin for this MB once. Already-present entries + /// are preserved so the first observer of this MB wins. + pub fn set_mb_origin_once(&self, block_hash: H256, origin: CallbackOrigin) -> Result<()> { + let key = Self::key_mb_origin(block_hash); + if self.db.get(&key)?.is_none() { + self.db.put(key, origin.encode())?; + } + Ok(()) + } + + /// Load the callback origin associated with this MB, if present. + pub fn mb_origin(&self, block_hash: H256) -> Result> { + match self.db.get(Self::key_mb_origin(block_hash))? { + Some(b) => Ok(Some(Self::decode_one::( + &b, + "mb callback origin", + )?)), + None => Ok(None), + } + } + /// Mark the block as `saved`. Idempotent. Errors if the block /// isn't in the store yet — the caller must `insert_block` first. pub fn mark_saved(&self, block_hash: H256) -> Result<()> { diff --git a/ethexe/malachite/core/tests/multi_validators.rs b/ethexe/malachite/core/tests/multi_validators.rs index 6c7330394f3..0cde79114f8 100644 --- a/ethexe/malachite/core/tests/multi_validators.rs +++ b/ethexe/malachite/core/tests/multi_validators.rs @@ -115,7 +115,12 @@ impl TestExt { #[async_trait] impl Externalities for TestExt { - async fn process_mb_proposal(&self, hash: H256, block: Block) -> Result<()> { + async fn process_mb_proposal( + &self, + hash: H256, + block: Block, + _origin: ethexe_malachite_core::CallbackOrigin, + ) -> Result<()> { let mut s = self.state.lock().unwrap(); if block.hash() != hash { s.violations @@ -136,7 +141,12 @@ impl Externalities for TestExt { Ok(()) } - async fn process_mb_finalized(&self, hash: H256, cert: CommitCertificate) -> Result<()> { + async fn process_mb_finalized( + &self, + hash: H256, + cert: CommitCertificate, + _origin: ethexe_malachite_core::CallbackOrigin, + ) -> Result<()> { let mut s = self.state.lock().unwrap(); if cert.block_hash != hash { s.violations diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index c3fa9ae7a3d..34b13344757 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -51,7 +51,7 @@ use ethexe_common::{ malachite::{ProcessQueuesLimits, ProgressTasksLimits, Transaction, Transactions}, }; use ethexe_db::Database; -use ethexe_malachite_core::{Block, Externalities}; +use ethexe_malachite_core::{Block, CallbackOrigin, Externalities}; use gprimitives::H256; use parity_scale_codec::Encode; use std::{ @@ -61,6 +61,17 @@ use std::{ use tokio::sync::{Notify, mpsc}; use tracing::{error, info, warn}; +#[derive(Clone, Copy)] +pub(crate) enum ReplayBoundary { + /// No replay boundary configured (both hashes are zero). + Disabled, + /// Stable replay suppression applies for MBs at or below this height. + AtHeight(u64), + /// Boundary hash is known, but compact is not yet available. + /// Suppress replay callbacks until the boundary MB itself is observed. + AtHash(H256), +} + /// Inputs the externalities need to satisfy the [`ethexe_malachite_core::Externalities`] /// contract. Constructed by [`crate::MalachiteService::new`] and /// handed to the inner ethexe-malachite-core service inside an [`Arc`]. @@ -100,6 +111,9 @@ pub(crate) struct EthexeExternalities { /// time they see the MB. Validators do NOT apply this depth — they accept /// any advance at depth ≥ `canonical_quarantine`. pub(crate) post_quarantine_delay: u32, + /// Captured once per externalities instance; prevents mutable live + /// computed progression from changing replay suppression scope. + pub(crate) replay_boundary: RwLock, } /// One outbound [`MalachiteEvent`] that can't be released until its @@ -115,7 +129,12 @@ pub(crate) struct PendingEvent { #[async_trait] impl Externalities for EthexeExternalities { - async fn process_mb_proposal(&self, mb_hash: H256, mb: Block) -> Result<()> { + async fn process_mb_proposal( + &self, + mb_hash: H256, + mb: Block, + origin: CallbackOrigin, + ) -> Result<()> { let parent = mb.parent_hash; let payload = mb.payload; @@ -152,6 +171,12 @@ impl Externalities for EthexeExternalities { meta.last_advanced_eb = last_advanced; }); + if origin == CallbackOrigin::ValueSyncReplay + && self.is_replay_mb_below_or_at_execution_boundary(mb_hash, mb.height) + { + return Ok(()); + } + self.try_emit_or_queue( MalachiteEvent::BlockProposal { height: mb.height, @@ -166,6 +191,7 @@ impl Externalities for EthexeExternalities { &self, mb_hash: H256, cert: ethexe_malachite_core::CommitCertificate, + origin: CallbackOrigin, ) -> Result<()> { let compact = self.db.mb_compact_block(mb_hash).ok_or_else(|| { anyhow!( @@ -183,6 +209,12 @@ impl Externalities for EthexeExternalities { ) })?; + if origin == CallbackOrigin::ValueSyncReplay + && self.is_replay_mb_below_or_at_execution_boundary(mb_hash, compact.height) + { + return Ok(()); + } + // Flush the committed injected txs from the mempool and add // their hashes to the seen-set so a re-gossip can't slip them // back in before they age out. @@ -594,6 +626,71 @@ impl Externalities for EthexeExternalities { } impl EthexeExternalities { + /// Whether `mb_hash`/`mb_height` are at or below the execution replay + /// boundary. + /// + /// Capture once per externalities instance and keep stable even if + /// live compute advances globals in between replay callbacks. + fn is_replay_mb_below_or_at_execution_boundary(&self, mb_hash: H256, mb_height: u64) -> bool { + let mut replay_boundary = self + .replay_boundary + .write() + .expect("replay_boundary poisoned"); + + match *replay_boundary { + ReplayBoundary::Disabled => return false, + ReplayBoundary::AtHeight(height) => return mb_height <= height, + ReplayBoundary::AtHash(boundary_hash) => { + if mb_hash != boundary_hash { + return true; + } + + if let Some(boundary_compact) = self.db.mb_compact_block(boundary_hash) { + *replay_boundary = ReplayBoundary::AtHeight(boundary_compact.height); + return mb_height <= boundary_compact.height; + } + return true; + } + } + } + + /// Capture a replay boundary from DB globals, preferring + /// `latest_computed_mb_hash` and falling back to + /// `latest_finalized_mb_hash`. + /// + /// This method intentionally does not mutate globals and does not + /// infer a boundary from any other signal. If no execution boundary + /// is available, suppression stays disabled. + fn read_replay_boundary_from_db(&self) -> ReplayBoundary { + let boundary_hash = { + let globals = self.db.globals(); + if !globals.latest_computed_mb_hash.is_zero() { + globals.latest_computed_mb_hash + } else { + globals.latest_finalized_mb_hash + } + }; + + if boundary_hash.is_zero() { + return ReplayBoundary::Disabled; + } + + self.db.mb_compact_block(boundary_hash).map_or_else( + || ReplayBoundary::AtHash(boundary_hash), + |boundary_compact| ReplayBoundary::AtHeight(boundary_compact.height), + ) + } + + /// Enable replay-boundary suppression from the current execution snapshot + /// persisted in DB globals. + pub(crate) fn enable_replay_boundary_from_execution_snapshot(&self) { + let replay_boundary = self.read_replay_boundary_from_db(); + *self + .replay_boundary + .write() + .expect("replay_boundary poisoned") = replay_boundary; + } + /// True iff `prerequisite.is_zero()` (no prerequisite — genesis /// or pre-advance) or the prerequisite Eth block has been fully /// **prepared** locally. @@ -726,6 +823,7 @@ mod tests { injected::PurgedTransaction, malachite::{ProcessQueuesLimits, ProgressTasksLimits}, }; + use ethexe_malachite_core::CallbackOrigin; /// Build a small ethexe `Database`-backed externalities + the /// matching event receiver. No ethexe-malachite-core or libp2p involved — @@ -748,6 +846,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 0, post_quarantine_delay: 0, + replay_boundary: RwLock::new(ReplayBoundary::Disabled), }; (ext, event_rx) } @@ -805,7 +904,9 @@ mod tests { let p = payload(None, 1); let block = wrap(p.clone(), 1, H256::zero()); let mb_hash = block.hash(); - ext.process_mb_proposal(mb_hash, block).await.unwrap(); + ext.process_mb_proposal(mb_hash, block, CallbackOrigin::Live) + .await + .unwrap(); let compact = db.mb_compact_block(mb_hash).expect("CompactMb saved"); assert_eq!(compact.height, 1); @@ -843,9 +944,11 @@ mod tests { let p = payload(None, 5); let block = wrap(p.clone(), 1, H256::zero()); let mb_hash = block.hash(); - ext.process_mb_proposal(mb_hash, block).await.unwrap(); + ext.process_mb_proposal(mb_hash, block, CallbackOrigin::Live) + .await + .unwrap(); let _ = rx.recv().await; // BlockProposal - ext.process_mb_finalized(mb_hash, fake_cert(1)) + ext.process_mb_finalized(mb_hash, fake_cert(1), CallbackOrigin::Live) .await .unwrap(); assert_eq!(db.globals().latest_finalized_mb_hash, mb_hash); @@ -881,9 +984,12 @@ mod tests { let p = payload(None, i as u8); let block = wrap(p.clone(), i, parent); let mb_hash = block.hash(); - ext_a.process_mb_proposal(mb_hash, block).await.unwrap(); ext_a - .process_mb_finalized(mb_hash, fake_cert(i)) + .process_mb_proposal(mb_hash, block, CallbackOrigin::Live) + .await + .unwrap(); + ext_a + .process_mb_finalized(mb_hash, fake_cert(i), CallbackOrigin::Live) .await .unwrap(); chain.push((mb_hash, p)); @@ -913,9 +1019,15 @@ mod tests { let p4 = payload(None, 99); let block4 = wrap(p4.clone(), 4, last_pre); let mb4 = block4.hash(); - ext_b.process_mb_proposal(mb4, block4).await.unwrap(); + ext_b + .process_mb_proposal(mb4, block4, CallbackOrigin::Live) + .await + .unwrap(); let _ = rx_b.recv().await; // proposal - ext_b.process_mb_finalized(mb4, fake_cert(4)).await.unwrap(); + ext_b + .process_mb_finalized(mb4, fake_cert(4), CallbackOrigin::Live) + .await + .unwrap(); assert_eq!(db.mb_compact_block(mb4).unwrap().parent, last_pre); assert_eq!(db.globals().latest_finalized_mb_hash, mb4); } @@ -940,8 +1052,10 @@ mod tests { let height = (i + 1) as u64; let block = wrap(p.clone(), height, parent); let mb_hash = block.hash(); - ext.process_mb_proposal(mb_hash, block).await.unwrap(); - ext.process_mb_finalized(mb_hash, fake_cert(height)) + ext.process_mb_proposal(mb_hash, block, CallbackOrigin::Live) + .await + .unwrap(); + ext.process_mb_finalized(mb_hash, fake_cert(height), CallbackOrigin::Live) .await .unwrap(); chain.push(mb_hash); @@ -1149,6 +1263,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 0, post_quarantine_delay: 0, + replay_boundary: RwLock::new(ReplayBoundary::Disabled), }; let payload = Transactions::new(vec![ @@ -1167,7 +1282,9 @@ mod tests { ]); let block = ethexe_malachite_core::Block::new(H256::zero(), 1, payload); let mb_hash = block.hash(); - ext.process_mb_proposal(mb_hash, block).await.unwrap(); + ext.process_mb_proposal(mb_hash, block, CallbackOrigin::Live) + .await + .unwrap(); // Drain the BlockProposal event the save emits. let _ = event_rx.recv().await; ext.process_mb_finalized( @@ -1177,6 +1294,7 @@ mod tests { block_hash: mb_hash, signatures: vec![], }, + CallbackOrigin::Live, ) .await .unwrap(); @@ -1192,6 +1310,494 @@ mod tests { assert!(seen_hashes.contains(&tx_b.data().to_hash())); } + /// Nonzero execution globals are not enough to suppress ValueSync replay. + /// Suppression must be explicitly enabled after fast-sync capture. + #[tokio::test] + async fn value_sync_replay_without_enabled_boundary_processes_normally() { + let db = Database::memory(); + let (ext, mut rx) = make_externalities(db.clone()); + + let boundary_payload = payload(None, 1); + let boundary_block = wrap(boundary_payload, 10, H256::zero()); + let boundary_hash = boundary_block.hash(); + db.globals_mutate(|g| { + g.latest_finalized_mb_hash = boundary_hash; + g.latest_computed_mb_hash = boundary_hash; + }); + + let replay_payload = payload(None, 2); + let replay_block = wrap(replay_payload, 9, H256::zero()); + let replay_hash = replay_block.hash(); + + ext.process_mb_proposal(replay_hash, replay_block, CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + match rx.recv().await.expect("proposal event").unwrap() { + MalachiteEvent::BlockProposal { height, mb_hash } => { + assert_eq!(height, 9); + assert_eq!(mb_hash, replay_hash); + } + other => panic!("expected BlockProposal, got {other:?}"), + } + + ext.process_mb_finalized(replay_hash, fake_cert(9), CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + match rx.recv().await.expect("finalization event").unwrap() { + MalachiteEvent::BlockFinalized { + cert, + height, + mb_hash, + } => { + assert_eq!(height, 9); + assert_eq!(cert.height, 9); + assert_eq!(mb_hash, replay_hash); + } + other => panic!("expected BlockFinalized, got {other:?}"), + } + assert_eq!(db.globals().latest_finalized_mb_hash, replay_hash); + } + + /// MBs at or below the execution boundary must persist MB + /// identity rows but never emit service events or retreat finalized + /// globals. + #[tokio::test] + async fn replay_below_execution_boundary_does_not_emit_or_retreat_globals() { + use ethexe_common::db::MbStorageRO; + + let db = Database::memory(); + let (ext, mut rx) = make_externalities(db.clone()); + + let boundary_payload = payload(None, 1); + let boundary_block = wrap(boundary_payload, 10, H256::zero()); + let boundary_hash = boundary_block.hash(); + db.globals_mutate(|g| { + g.latest_finalized_mb_hash = boundary_hash; + g.latest_computed_mb_hash = boundary_hash; + }); + ext.enable_replay_boundary_from_execution_snapshot(); + + ext.process_mb_proposal( + boundary_hash, + boundary_block, + CallbackOrigin::ValueSyncReplay, + ) + .await + .unwrap(); + assert!( + rx.try_recv().is_err(), + "boundary replay should be suppressed" + ); + assert!( + ext.pending_events + .lock() + .expect("pending_events poisoned") + .is_empty(), + "boundary replay should not queue events", + ); + + let old_eb = H256::repeat_byte(0xA1); + let replay_payload = Transactions::new(vec![ + Transaction::AdvanceTillEthereumBlock { block_hash: old_eb }, + Transaction::ProgressTasks { + limits: ProgressTasksLimits::default(), + }, + Transaction::ProcessQueues { + limits: ProcessQueuesLimits::default(), + }, + ]); + let replay_block = wrap(replay_payload, 10, H256::zero()); + let replay_hash = replay_block.hash(); + + ext.process_mb_proposal(replay_hash, replay_block, CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + assert!(rx.try_recv().is_err(), "proposal replay must not emit"); + assert!( + ext.pending_events + .lock() + .expect("pending_events poisoned") + .is_empty(), + "old replay must not queue service events", + ); + + assert_eq!(db.mb_meta(replay_hash).last_advanced_eb, old_eb); + let replay_compact = db + .mb_compact_block(replay_hash) + .expect("replay compact persisted"); + assert_eq!(replay_compact.height, 10); + assert!(db.transactions(replay_compact.transactions_hash).is_some()); + + ext.process_mb_finalized(replay_hash, fake_cert(10), CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + assert_eq!(db.globals().latest_finalized_mb_hash, boundary_hash); + assert!(rx.try_recv().is_err(), "finalization replay must not emit"); + assert!( + ext.pending_events + .lock() + .expect("pending_events poisoned") + .is_empty(), + "old replay must not enqueue pending service events", + ); + } + + /// Post-boundary MBs should keep normal service event behavior. + #[tokio::test] + async fn post_boundary_mb_emits_events_normally() { + let db = Database::memory(); + let (ext, mut rx) = make_externalities(db.clone()); + + let boundary_payload = payload(None, 1); + let boundary_block = wrap(boundary_payload, 10, H256::zero()); + let boundary_hash = boundary_block.hash(); + db.globals_mutate(|g| { + g.latest_finalized_mb_hash = boundary_hash; + g.latest_computed_mb_hash = boundary_hash; + }); + ext.enable_replay_boundary_from_execution_snapshot(); + + ext.process_mb_proposal( + boundary_hash, + boundary_block, + CallbackOrigin::ValueSyncReplay, + ) + .await + .unwrap(); + assert!( + rx.try_recv().is_err(), + "boundary replay should be suppressed" + ); + assert!( + ext.pending_events + .lock() + .expect("pending_events poisoned") + .is_empty(), + "boundary replay should not queue events", + ); + + let advanced_eb = H256::repeat_byte(0xA2); + db.mutate_block_meta(advanced_eb, |meta| { + meta.prepared = true; + }); + + let replay_payload = Transactions::new(vec![ + Transaction::AdvanceTillEthereumBlock { + block_hash: advanced_eb, + }, + Transaction::ProgressTasks { + limits: ProgressTasksLimits::default(), + }, + Transaction::ProcessQueues { + limits: ProcessQueuesLimits::default(), + }, + ]); + let replay_block = wrap(replay_payload, 11, boundary_hash); + let replay_hash = replay_block.hash(); + + ext.process_mb_proposal(replay_hash, replay_block, CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + assert_eq!( + ext.pending_events + .lock() + .expect("pending_events poisoned") + .len(), + 0 + ); + match rx.recv().await.expect("proposal event").unwrap() { + MalachiteEvent::BlockProposal { height, mb_hash } => { + assert_eq!(height, 11); + assert_eq!(mb_hash, replay_hash); + } + other => panic!("expected BlockProposal, got {other:?}"), + } + + ext.process_mb_finalized(replay_hash, fake_cert(11), CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + match rx.recv().await.expect("finalization event").unwrap() { + MalachiteEvent::BlockFinalized { + cert, + height, + mb_hash, + } => { + assert_eq!(height, 11); + assert_eq!(cert.height, 11); + assert_eq!(mb_hash, replay_hash); + } + other => panic!("expected BlockFinalized, got {other:?}"), + } + assert_eq!(db.globals().latest_finalized_mb_hash, replay_hash); + } + + /// If the execution boundary compact is still missing, suppress replay + /// callbacks by hash until the boundary MB is observed. + #[tokio::test] + async fn replay_with_missing_boundary_compact_suppresses_until_boundary_is_seen() { + let db = Database::memory(); + let (ext, mut rx) = make_externalities(db.clone()); + + let boundary_payload = payload(None, 2); + let boundary_block = wrap(boundary_payload, 10, H256::zero()); + let boundary_hash = boundary_block.hash(); + + db.globals_mutate(|g| { + g.latest_finalized_mb_hash = boundary_hash; + g.latest_computed_mb_hash = boundary_hash; + }); + ext.enable_replay_boundary_from_execution_snapshot(); + + let old_eb = H256::repeat_byte(0xB1); + let old_payload = Transactions::new(vec![ + Transaction::AdvanceTillEthereumBlock { block_hash: old_eb }, + Transaction::ProgressTasks { + limits: ProgressTasksLimits::default(), + }, + Transaction::ProcessQueues { + limits: ProcessQueuesLimits::default(), + }, + ]); + let old_block = wrap(old_payload, 9, H256::zero()); + let old_hash = old_block.hash(); + + ext.process_mb_proposal(old_hash, old_block, CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + ext.process_mb_finalized(old_hash, fake_cert(9), CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + + assert!(rx.try_recv().is_err(), "replay should be suppressed"); + assert!( + ext.pending_events + .lock() + .expect("pending_events poisoned") + .is_empty(), + "replay should not enqueue pending service events", + ); + assert_eq!( + db.globals().latest_finalized_mb_hash, + boundary_hash, + "replay finalization must not retreat finalized boundary", + ); + + // Boundary MB itself is still missing in compact storage when replay starts. + // Process it now to both persist consensus identity and switch boundary mode. + ext.process_mb_proposal( + boundary_hash, + boundary_block, + CallbackOrigin::ValueSyncReplay, + ) + .await + .unwrap(); + assert!( + rx.try_recv().is_err(), + "boundary MB is still replay context" + ); + + let advanced_eb = H256::repeat_byte(0xB2); + db.mutate_block_meta(advanced_eb, |meta| { + meta.prepared = true; + }); + + let post_boundary_payload = Transactions::new(vec![ + Transaction::AdvanceTillEthereumBlock { + block_hash: advanced_eb, + }, + Transaction::ProgressTasks { + limits: ProgressTasksLimits::default(), + }, + Transaction::ProcessQueues { + limits: ProcessQueuesLimits::default(), + }, + ]); + let post_boundary_block = wrap(post_boundary_payload, 11, boundary_hash); + let post_boundary_hash = post_boundary_block.hash(); + + ext.process_mb_proposal( + post_boundary_hash, + post_boundary_block, + CallbackOrigin::ValueSyncReplay, + ) + .await + .unwrap(); + match rx.recv().await.expect("proposal event").unwrap() { + MalachiteEvent::BlockProposal { height, mb_hash } => { + assert_eq!(height, 11); + assert_eq!(mb_hash, post_boundary_hash); + } + other => panic!("expected BlockProposal, got {other:?}"), + } + + ext.process_mb_finalized( + post_boundary_hash, + fake_cert(11), + CallbackOrigin::ValueSyncReplay, + ) + .await + .unwrap(); + match rx.recv().await.expect("finalization event").unwrap() { + MalachiteEvent::BlockFinalized { + height, mb_hash, .. + } => { + assert_eq!(height, 11); + assert_eq!(mb_hash, post_boundary_hash); + } + other => panic!("expected BlockFinalized, got {other:?}"), + } + assert_eq!(db.globals().latest_finalized_mb_hash, post_boundary_hash); + } + + /// When the boundary compact is missing, old replay callbacks are + /// still suppressed, but live callbacks for post-boundary MBs must + /// continue to emit and advance globals. + #[tokio::test] + async fn live_callbacks_not_suppressed_when_boundary_compact_is_missing() { + let db = Database::memory(); + let (ext, mut rx) = make_externalities(db.clone()); + + let boundary_payload = payload(None, 2); + let boundary_block = wrap(boundary_payload, 10, H256::zero()); + let boundary_hash = boundary_block.hash(); + db.globals_mutate(|g| { + g.latest_finalized_mb_hash = boundary_hash; + g.latest_computed_mb_hash = boundary_hash; + }); + ext.enable_replay_boundary_from_execution_snapshot(); + + let replay_payload = Transactions::new(vec![ + Transaction::ProgressTasks { + limits: ProgressTasksLimits::default(), + }, + Transaction::ProcessQueues { + limits: ProcessQueuesLimits::default(), + }, + ]); + let old_block = wrap(replay_payload, 9, H256::zero()); + let old_hash = old_block.hash(); + ext.process_mb_proposal(old_hash, old_block, CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + assert!(rx.try_recv().is_err(), "replay should be suppressed"); + + let live_payload = Transactions::new(vec![ + Transaction::ProgressTasks { + limits: ProgressTasksLimits::default(), + }, + Transaction::ProcessQueues { + limits: ProcessQueuesLimits::default(), + }, + ]); + let live_block = wrap(live_payload, 11, boundary_hash); + let live_hash = live_block.hash(); + ext.process_mb_proposal(live_hash, live_block, CallbackOrigin::Live) + .await + .unwrap(); + match rx.recv().await.expect("proposal event").unwrap() { + MalachiteEvent::BlockProposal { height, mb_hash } => { + assert_eq!(height, 11); + assert_eq!(mb_hash, live_hash); + } + other => panic!("expected BlockProposal, got {other:?}"), + } + + ext.process_mb_finalized(live_hash, fake_cert(11), CallbackOrigin::Live) + .await + .unwrap(); + match rx.recv().await.expect("finalization event").unwrap() { + MalachiteEvent::BlockFinalized { + cert, + height, + mb_hash, + } => { + assert_eq!(height, 11); + assert_eq!(cert.height, 11); + assert_eq!(mb_hash, live_hash); + } + other => panic!("expected BlockFinalized, got {other:?}"), + } + assert_eq!(db.globals().latest_finalized_mb_hash, live_hash); + } + + /// Live recomputation progress must not make the replay boundary + /// move forward before finalization processing. + #[tokio::test] + async fn computed_tip_advancing_does_not_disable_post_boundary_finalization_events() { + let db = Database::memory(); + let (ext, mut rx) = make_externalities(db.clone()); + + let boundary_payload = payload(None, 1); + let boundary_block = wrap(boundary_payload, 10, H256::zero()); + let boundary_hash = boundary_block.hash(); + db.globals_mutate(|g| { + g.latest_finalized_mb_hash = boundary_hash; + g.latest_computed_mb_hash = boundary_hash; + }); + ext.enable_replay_boundary_from_execution_snapshot(); + + ext.process_mb_proposal( + boundary_hash, + boundary_block, + CallbackOrigin::ValueSyncReplay, + ) + .await + .unwrap(); + assert!( + rx.try_recv().is_err(), + "boundary replay should be suppressed" + ); + + let advanced_eb = H256::repeat_byte(0xA3); + db.mutate_block_meta(advanced_eb, |meta| { + meta.prepared = true; + }); + + let replay_payload = Transactions::new(vec![ + Transaction::AdvanceTillEthereumBlock { + block_hash: advanced_eb, + }, + Transaction::ProgressTasks { + limits: ProgressTasksLimits::default(), + }, + Transaction::ProcessQueues { + limits: ProcessQueuesLimits::default(), + }, + ]); + let replay_block = wrap(replay_payload, 11, boundary_hash); + let replay_hash = replay_block.hash(); + + ext.process_mb_proposal(replay_hash, replay_block, CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + match rx.recv().await.expect("proposal event").unwrap() { + MalachiteEvent::BlockProposal { height, mb_hash } => { + assert_eq!(height, 11); + assert_eq!(mb_hash, replay_hash); + } + other => panic!("expected BlockProposal, got {other:?}"), + } + + db.globals_mutate(|g| g.latest_computed_mb_hash = replay_hash); + + ext.process_mb_finalized(replay_hash, fake_cert(11), CallbackOrigin::ValueSyncReplay) + .await + .unwrap(); + match rx.recv().await.expect("finalization event").unwrap() { + MalachiteEvent::BlockFinalized { + cert, + height, + mb_hash, + } => { + assert_eq!(height, 11); + assert_eq!(cert.height, 11); + assert_eq!(mb_hash, replay_hash); + } + other => panic!("expected BlockFinalized, got {other:?}"), + } + assert_eq!(db.globals().latest_finalized_mb_hash, replay_hash); + } + // ------------------------------------------------------------------ // Integration tests for build_block_above / validate_block_above // size + touched-programs caps. Adapted from master's @@ -1219,6 +1825,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 0, post_quarantine_delay: 0, + replay_boundary: RwLock::new(ReplayBoundary::Disabled), }; (ext, event_rx) } @@ -1998,6 +2605,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 2, post_quarantine_delay: 3, + replay_boundary: RwLock::new(ReplayBoundary::Disabled), }; let candidate = ext diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 56684b20e84..55c855e81b6 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -36,7 +36,8 @@ use gsigner::{Signer, schemes::secp256k1::Secp256k1}; use tokio::sync::{Notify, mpsc}; use crate::{ - MalachiteConfig, MalachiteEvent, Mempool, ValidatorEntry, externalities::EthexeExternalities, + MalachiteConfig, MalachiteEvent, Mempool, ValidatorEntry, + externalities::{EthexeExternalities, ReplayBoundary}, }; /// Public consensus service. @@ -166,6 +167,7 @@ impl MalachiteService { gas_allowance: config.gas_allowance, canonical_quarantine: config.canonical_quarantine, post_quarantine_delay: config.post_quarantine_delay, + replay_boundary: std::sync::RwLock::new(ReplayBoundary::Disabled), }); // On-chain addresses → pub keys, so era rotations resolve back without an out-of-band lookup. @@ -198,6 +200,14 @@ impl MalachiteService { } } + /// Capture the execution-layer replay boundary from DB globals so + /// `ValueSyncReplay` callbacks can be suppressed for pre-boundary + /// Malachite MBs after fast-sync. + pub fn enable_replay_boundary_from_execution_snapshot(&self) { + self.externalities + .enable_replay_boundary_from_execution_snapshot(); + } + /// Hand an injected transaction to the mempool. The local /// producer pulls from the same pool when assembling the next MB. /// diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index db2d04245be..51786e15253 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -9,15 +9,14 @@ use anyhow::{Context, Result, ensure}; use ethexe_common::{ Address, BlockData, CodeAndIdUnchecked, Digest, ProgramStates, StateHashWithQueueSize, db::{ - BlockMetaStorageRO, CodesStorageRO, CodesStorageRW, CompactMb, ConfigStorageRO, - GlobalsStorageRW, MbStorageRW, OnChainStorageRW, PreparedBlockData, + BlockMetaStorageRO, CodesStorageRO, CodesStorageRW, ConfigStorageRO, GlobalsStorageRW, + MbStorageRW, OnChainStorageRW, PreparedBlockData, }, events::{ BlockEvent, RouterEvent, router::{BatchCommittedEvent, EBCommittedEvent, MBCommittedEvent}, }, injected, - malachite::Transactions, }; use ethexe_compute::ComputeService; use ethexe_db::{ @@ -587,7 +586,7 @@ async fn latest_era_with_committed_validators( .context("failed to calculate era from validators timestamp") } -pub(crate) async fn sync(service: &mut Service) -> Result<()> { +pub(crate) async fn sync(service: &mut Service) -> Result { let Service { observer, compute, @@ -600,7 +599,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { } = service; let Some(network) = network else { log::warn!("Network service is disabled. Skipping fast synchronization..."); - return Ok(()); + return Ok(false); }; log::info!("Fast synchronization is in progress..."); @@ -638,7 +637,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { EventData::collect(&block_loader, db, latest_block).await? else { log::warn!("No committed MB found. Skipping fast synchronization..."); - return Ok(()); + return Ok(false); }; log::warn!("No finalized committed MB found; trying latest block candidates"); state_query_block = latest_block; @@ -646,7 +645,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { } None => { log::warn!("No committed MB found. Skipping fast synchronization..."); - return Ok(()); + return Ok(false); } }; @@ -747,5 +746,5 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { )) .await; - Ok(()) + Ok(true) } diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 09e75fd0193..2891d1283b2 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -580,7 +580,11 @@ impl Service { pub async fn run(mut self) -> Result<()> { if self.fast_sync { - fast_sync::sync(&mut self).await?; + if fast_sync::sync(&mut self).await? + && let Some(malachite) = &self.malachite + { + malachite.enable_replay_boundary_from_execution_snapshot(); + } } self.run_inner().await.inspect_err(|err| { From 3bf6a6f2604161dcc78be08cce45302f46195833 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 10 Jun 2026 15:51:09 +0300 Subject: [PATCH 24/72] Skip fast synced MB in chain assertion --- ethexe/service/src/fast_sync.rs | 7 +- ethexe/service/src/tests/mod.rs | 83 ++++++++++-------------- ethexe/service/src/tests/utils/env.rs | 17 +++-- ethexe/service/src/tests/utils/events.rs | 5 +- 4 files changed, 53 insertions(+), 59 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 51786e15253..2f8125ec9b9 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -741,9 +741,10 @@ pub(crate) async fn sync(service: &mut Service) -> Result { #[cfg(test)] sender - .send(crate::tests::utils::TestingEvent::FastSyncDone( - latest_committed_eb, - )) + .send(crate::tests::utils::TestingEvent::FastSyncDone { + eb_hash: latest_committed_eb, + mb_hash: latest_committed_mb, + }) .await; Ok(true) diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index a947cd4095e..b09274f9b96 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -6,9 +6,9 @@ pub(crate) mod utils; use crate::tests::utils::{ - EnvNetworkConfig, InfiniteStreamExt, Node, NodeConfig, TestEnv, TestEnvConfig, TestingEvent, - TestingEventReceiver, TestingNetworkEvent, TestingRpcEvent, ValidatorsConfig, init_logger, - stop_nodes, test_info, + EnvNetworkConfig, InfiniteStreamExt, LatestFastSyncedBlocks, Node, NodeConfig, TestEnv, + TestEnvConfig, TestingEvent, TestingEventReceiver, TestingNetworkEvent, TestingRpcEvent, + ValidatorsConfig, init_logger, stop_nodes, test_info, }; use alloy::{ primitives::U256, @@ -3250,20 +3250,25 @@ async fn fast_sync2() { // } // }; - let assert_chain = |latest_block, fast_synced_block, alice: &Node, bob: &Node| { - log::info!("Assert chain in range {latest_block}..{fast_synced_block}"); + let assert_chain = |latest_block, fast_synced_blocks, alice: &Node, bob: &Node| { + let LatestFastSyncedBlocks { + eb_hash: fast_synced_eb, + mb_hash: fast_synced_mb, + } = fast_synced_blocks; + + log::info!("Assert chain in range {latest_block}..{fast_synced_eb}"); IntegrityVerifier::new(alice.db.clone()) - .verify_chain(latest_block, fast_synced_block) + .verify_chain(latest_block, fast_synced_eb) .expect("failed to verify Alice database"); IntegrityVerifier::new(bob.db.clone()) - .verify_chain(latest_block, fast_synced_block) + .verify_chain(latest_block, fast_synced_eb) .expect("failed to verify Bob database"); let mut eb_hash = latest_block; loop { - if fast_synced_block == eb_hash { + if fast_synced_eb == eb_hash { break; } @@ -3272,38 +3277,31 @@ async fn fast_sync2() { // Check block meta, exclude codes_queue and announces, which can vary, and it's ok let alice_meta = alice.db.block_meta(eb_hash); let bob_meta = bob.db.block_meta(eb_hash); - assert_eq!(alice_meta, bob_meta); - // assert!( - // alice_meta.prepared && bob_meta.prepared, - // "Block {block} is not prepared for alice or bob" - // ); - // assert_eq!( - // alice_meta.last_committed_announce, - // bob_meta.last_committed_announce - // ); - // assert_eq!( - // alice_meta.last_committed_batch, - // bob_meta.last_committed_batch - // ); + assert_eq!(alice_meta.prepared, bob_meta.prepared); + assert_eq!( + alice_meta.latest_era_validators_committed, + bob_meta.latest_era_validators_committed + ); assert_eq!(alice.db.block_header(eb_hash), bob.db.block_header(eb_hash)); assert_eq!(alice.db.block_events(eb_hash), bob.db.block_events(eb_hash)); assert_eq!(alice.db.block_synced(eb_hash), bob.db.block_synced(eb_hash)); + let header = alice.db.block_header(eb_hash).unwrap(); + eb_hash = header.parent_hash; + let mb_hash = bob_meta.last_committed_mb.unwrap(); + if mb_hash == fast_synced_mb { + continue; + } - let alice_compact_block = alice.db.mb_compact_block(mb_hash); - let bob_compact_block = bob.db.mb_compact_block(mb_hash); + let alice_compact_block = alice.db.mb_compact_block(mb_hash).unwrap(); + let bob_compact_block = bob.db.mb_compact_block(mb_hash).unwrap(); assert_eq!(alice_compact_block, bob_compact_block); - if let (Some(alice_compact_block), Some(bob_compact_block)) = - (alice_compact_block, bob_compact_block) - { - let alice_transactions = - alice.db.transactions(alice_compact_block.transactions_hash); - let bob_transactions = bob.db.transactions(bob_compact_block.transactions_hash); - assert_eq!(alice_transactions, bob_transactions); - } + let alice_transactions = alice.db.transactions(alice_compact_block.transactions_hash); + let bob_transactions = bob.db.transactions(bob_compact_block.transactions_hash); + assert_eq!(alice_transactions, bob_transactions); let alice_program_states = alice.db.mb_program_states(mb_hash); let bob_program_states = bob.db.mb_program_states(mb_hash); @@ -3316,13 +3314,6 @@ async fn fast_sync2() { let alice_schedule = alice.db.mb_schedule(mb_hash); let bob_schedule = bob.db.mb_schedule(mb_hash); assert_eq!(alice_schedule, bob_schedule); - - let alice_meta = alice.db.mb_meta(mb_hash); - let bob_meta = bob.db.mb_meta(mb_hash); - assert_eq!(alice_meta, bob_meta); - - let header = alice.db.block_header(eb_hash).unwrap(); - eb_hash = header.parent_hash; } // let mut block_hash = latest_block; @@ -3450,17 +3441,13 @@ async fn fast_sync2() { alice_events.find_advanced_mb(latest_block), bob_events.find_advanced_mb(latest_block) ); - tokio::join!( - alice_events.find_mb_computed(alice_mb_hash), - bob_events.find_mb_computed(bob_mb_hash) - ); test_info!("📗 Stopping Bob"); bob.stop_service().await; assert_chain( latest_block, - bob.latest_fast_synced_block.take().unwrap(), + bob.latest_fast_synced_blocks.unwrap(), &alice, &bob, ); @@ -3505,7 +3492,7 @@ async fn fast_sync2() { assert_chain( latest_block, - bob.latest_fast_synced_block.take().unwrap(), + bob.latest_fast_synced_blocks.unwrap(), &alice, &bob, ); @@ -3742,9 +3729,9 @@ async fn fast_sync() { .await .expect("Bob did not finish fast sync"); - let fast_synced_block = bob.latest_fast_synced_block.take().unwrap(); - assert!(!fast_synced_block.is_zero()); - let fast_synced_mb = assert_fast_sync_boundary(fast_synced_block, &validators[0], &bob); + let fast_synced_blocks = bob.latest_fast_synced_blocks.take().unwrap(); + let fast_synced_mb = + assert_fast_sync_boundary(fast_synced_blocks.eb_hash, &validators[0], &bob); test_info!("Sending post-fast-sync message and comparing Bob with Alice"); let mut alice_events = validators[0].events(); @@ -3763,7 +3750,7 @@ async fn fast_sync() { wait_till_block_prepared(&mut bob_events, latest_block).await; assert_chain( latest_block, - fast_synced_block, + fast_synced_blocks.eb_hash, fast_synced_mb, &validators[0], &bob, diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index ef8caca2b8b..c5fff3f2ab5 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -505,7 +505,7 @@ impl TestEnv { name, db, multiaddr: None, - latest_fast_synced_block: None, + latest_fast_synced_blocks: None, custom_committer: None, router_query: self.router_query.clone(), eth_cfg: self.eth_cfg.clone(), @@ -1004,11 +1004,17 @@ impl Wallets { } } +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub struct LatestFastSyncedBlocks { + pub eb_hash: H256, + pub mb_hash: H256, +} + pub struct Node { pub name: Option, pub db: Database, pub multiaddr: Option, - pub latest_fast_synced_block: Option, + pub latest_fast_synced_blocks: Option, pub custom_committer: Option>, router_query: RouterQuery, @@ -1278,13 +1284,14 @@ impl Node { .find(|event| { matches!( event, - TestingEvent::FastSyncDone(_) | TestingEvent::ServiceStarted + TestingEvent::FastSyncDone { .. } | TestingEvent::ServiceStarted ) }) .await { - TestingEvent::FastSyncDone(block) => { - self.latest_fast_synced_block = Some(block); + TestingEvent::FastSyncDone { eb_hash, mb_hash } => { + self.latest_fast_synced_blocks = + Some(LatestFastSyncedBlocks { eb_hash, mb_hash }); } TestingEvent::ServiceStarted => { service_started = true; diff --git a/ethexe/service/src/tests/utils/events.rs b/ethexe/service/src/tests/utils/events.rs index d1137d38f54..ddf67ecd8df 100644 --- a/ethexe/service/src/tests/utils/events.rs +++ b/ethexe/service/src/tests/utils/events.rs @@ -121,11 +121,10 @@ impl TestingRpcEvent { } } -#[derive(Debug, Clone, Eq, PartialEq, derive_more::TryUnwrap)] +#[derive(Debug, Clone, Eq, PartialEq)] pub enum TestingEvent { // Fast sync done. Sent just once. - #[allow(dead_code)] - FastSyncDone(H256), + FastSyncDone { eb_hash: H256, mb_hash: H256 }, // Basic event to notify that service has started. Sent just once. ServiceStarted, // Services events. From 9481cd4f7d24492d41a8503ffcbd3baab329fa6d Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 10 Jun 2026 16:17:01 +0300 Subject: [PATCH 25/72] Fixes after merge --- Cargo.lock | 1 - ethexe/malachite/core/src/app.rs | 26 ++--- ethexe/malachite/core/src/externalities.rs | 5 +- ethexe/malachite/core/src/store.rs | 3 +- ethexe/malachite/service/src/externalities.rs | 108 +++++++++--------- ethexe/service/src/fast_sync.rs | 3 +- ethexe/service/src/lib.rs | 11 +- ethexe/service/src/tests/mod.rs | 31 ++--- 8 files changed, 84 insertions(+), 104 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc977edc497..a1764d4160c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7863,7 +7863,6 @@ dependencies = [ "clap_builder", "concurrent-queue", "const-hex", - "core2", "cranelift-bitset", "cranelift-codegen", "cranelift-codegen-meta", diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index e58d02f54a5..65b0a5acd61 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -808,12 +808,7 @@ mod tests { #[async_trait] impl Externalities for NoopExt { - async fn process_mb_proposal( - &self, - _: H256, - _: Block, - _: CallbackOrigin, - ) -> Result<()> { + async fn process_mb_proposal(&self, _: H256, _: Block, _: CallbackOrigin) -> Result<()> { Ok(()) } async fn process_mb_finalized( @@ -824,10 +819,10 @@ mod tests { ) -> Result<()> { Ok(()) } - async fn build_block_above(&self, _: H256) -> Result { - Ok(TestPayload) + async fn build_block_above(&self, _: H256) -> Result { + Ok(test_payload()) } - async fn validate_block_above(&self, _: H256, _: TestPayload) -> Result { + async fn validate_block_above(&self, _: H256, _: BlockPayload) -> Result { Ok(true) } } @@ -859,7 +854,7 @@ mod tests { async fn process_mb_proposal( &self, _: H256, - _: Block, + _: Block, origin: CallbackOrigin, ) -> Result<()> { self.proposal_origins @@ -941,7 +936,7 @@ mod tests { let ext = OriginRecordingExt::default(); let handler_ext = ext.clone(); let (mut handler, _dir, proposer) = make_handler_with(1, handler_ext); - let block = Block::::new(H256::zero(), 1, TestPayload); + let block = Block::new(H256::zero(), 1, test_payload()); let block_bytes = block.encode(); let value_bytes = encode_value(&crate::context::Value::new(block_bytes.clone())); @@ -980,7 +975,7 @@ mod tests { let ext = OriginRecordingExt::default(); let handler_ext = ext.clone(); let (mut handler, _dir, address) = make_handler_with(1, handler_ext); - let block = Block::::new(H256::zero(), 1, TestPayload); + let block = Block::new(H256::zero(), 1, test_payload()); let stream = complete_stream(address, 1, &block.encode()); run_stream(&mut handler, test_peer(2), stream) @@ -1130,12 +1125,7 @@ mod tests { #[async_trait] impl Externalities for FailingFinalizeExt { - async fn process_mb_proposal( - &self, - _: H256, - _: Block, - _: CallbackOrigin, - ) -> Result<()> { + async fn process_mb_proposal(&self, _: H256, _: Block, _: CallbackOrigin) -> Result<()> { Ok(()) } async fn process_mb_finalized( diff --git a/ethexe/malachite/core/src/externalities.rs b/ethexe/malachite/core/src/externalities.rs index 22e95b8cf6c..e5408e174a6 100644 --- a/ethexe/malachite/core/src/externalities.rs +++ b/ethexe/malachite/core/src/externalities.rs @@ -3,11 +3,14 @@ //! Application callbacks the service makes to the outside world. +use crate::{ + BlockPayload, + types::{Block, CommitCertificate, H256}, +}; use anyhow::Result; use async_trait::async_trait; use parity_scale_codec::{Decode, Encode}; use serde::{Deserialize, Serialize}; -use crate::types::{Block, CommitCertificate, H256}; /// Indicates where a proposed/finalized MB callback originated. /// `CallbackOrigin::ValueSyncReplay` identifies callbacks produced by diff --git a/ethexe/malachite/core/src/store.rs b/ethexe/malachite/core/src/store.rs index 926a1ea673b..626a7c46c99 100644 --- a/ethexe/malachite/core/src/store.rs +++ b/ethexe/malachite/core/src/store.rs @@ -36,6 +36,7 @@ use parity_scale_codec::{Decode, Encode}; use rocksdb::{DB, Options, WriteBatch}; use crate::{ + BlockPayload, context::Height, externalities::CallbackOrigin, types::{Block, CommitCertificate, H256}, @@ -195,7 +196,7 @@ impl Store { /// are preserved so the first observer of this MB wins. pub fn set_mb_origin_once(&self, block_hash: H256, origin: CallbackOrigin) -> Result<()> { let key = Self::key_mb_origin(block_hash); - if self.db.get(&key)?.is_none() { + if self.db.get(key)?.is_none() { self.db.put(key, origin.encode())?; } Ok(()) diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index 7f530710a62..10ac67ed6ef 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -51,7 +51,9 @@ use ethexe_common::{ malachite::{Operation, Operations}, }; use ethexe_db::Database; -use ethexe_malachite_core::{Block, CallbackOrigin, Externalities, MAX_BLOCK_PAYLOAD_BYTES}; +use ethexe_malachite_core::{ + Block, BlockPayload, CallbackOrigin, Externalities, MAX_BLOCK_PAYLOAD_BYTES, +}; use gprimitives::H256; use parity_scale_codec::{DecodeAll, Encode}; use std::{ @@ -129,7 +131,12 @@ pub(crate) struct PendingEvent { #[async_trait] impl Externalities for EthexeExternalities { - async fn process_mb_proposal(&self, mb_hash: H256, mb: Block, origin: CallbackOrigin) -> Result<()> { + async fn process_mb_proposal( + &self, + mb_hash: H256, + mb: Block, + origin: CallbackOrigin, + ) -> Result<()> { // Runs on the proposer, participant, and sync paths. Frozen // discriminants mean every historical operation always decodes; an // unknown one can only come from a newer protocol this build doesn't @@ -651,8 +658,8 @@ impl EthexeExternalities { .expect("replay_boundary poisoned"); match *replay_boundary { - ReplayBoundary::Disabled => return false, - ReplayBoundary::AtHeight(height) => return mb_height <= height, + ReplayBoundary::Disabled => false, + ReplayBoundary::AtHeight(height) => mb_height <= height, ReplayBoundary::AtHash(boundary_hash) => { if mb_hash != boundary_hash { return true; @@ -662,7 +669,8 @@ impl EthexeExternalities { *replay_boundary = ReplayBoundary::AtHeight(boundary_compact.height); return mb_height <= boundary_compact.height; } - return true; + + true } } } @@ -832,7 +840,7 @@ mod tests { use crate::{MalachiteEvent, mempool::EmptyMempool}; use anyhow::Context; use ethexe_common::{ - BlockHeader, + BlockHeader, DEFAULT_BLOCK_GAS_LIMIT, db::{BlockMetaStorageRW, OnChainStorageRW}, injected::PurgedTransaction, }; @@ -1151,7 +1159,11 @@ mod tests { let (ext, _rx) = make_externalities(db.clone()); let block = Block::new(H256::zero(), 1, to_payload(vec![0xff, 0xff, 0xff, 0xff])); let mb_hash = block.hash(); - assert!(ext.process_mb_proposal(mb_hash, block).await.is_err()); + assert!( + ext.process_mb_proposal(mb_hash, block, CallbackOrigin::Live) + .await + .is_err() + ); } #[tokio::test] @@ -1438,13 +1450,11 @@ mod tests { ); let old_eb = H256::repeat_byte(0xA1); - let replay_payload = Transactions::new(vec![ - Transaction::AdvanceTillEthereumBlock { block_hash: old_eb }, - Transaction::ProgressTasks { - limits: ProgressTasksLimits::default(), - }, - Transaction::ProcessQueues { - limits: ProcessQueuesLimits::default(), + let replay_payload = Operations::new(vec![ + Operation::AdvanceTillEthereumBlock { block_hash: old_eb }, + Operation::ProgressTasks, + Operation::ProcessQueues { + gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, }, ]); let replay_block = wrap(replay_payload, 10, H256::zero()); @@ -1467,7 +1477,7 @@ mod tests { .mb_compact_block(replay_hash) .expect("replay compact persisted"); assert_eq!(replay_compact.height, 10); - assert!(db.transactions(replay_compact.transactions_hash).is_some()); + assert!(db.operations(replay_compact.operations_hash).is_some()); ext.process_mb_finalized(replay_hash, fake_cert(10), CallbackOrigin::ValueSyncReplay) .await @@ -1522,15 +1532,13 @@ mod tests { meta.prepared = true; }); - let replay_payload = Transactions::new(vec![ - Transaction::AdvanceTillEthereumBlock { + let replay_payload = Operations::new(vec![ + Operation::AdvanceTillEthereumBlock { block_hash: advanced_eb, }, - Transaction::ProgressTasks { - limits: ProgressTasksLimits::default(), - }, - Transaction::ProcessQueues { - limits: ProcessQueuesLimits::default(), + Operation::ProgressTasks, + Operation::ProcessQueues { + gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, }, ]); let replay_block = wrap(replay_payload, 11, boundary_hash); @@ -1590,13 +1598,11 @@ mod tests { ext.enable_replay_boundary_from_execution_snapshot(); let old_eb = H256::repeat_byte(0xB1); - let old_payload = Transactions::new(vec![ - Transaction::AdvanceTillEthereumBlock { block_hash: old_eb }, - Transaction::ProgressTasks { - limits: ProgressTasksLimits::default(), - }, - Transaction::ProcessQueues { - limits: ProcessQueuesLimits::default(), + let old_payload = Operations::new(vec![ + Operation::AdvanceTillEthereumBlock { block_hash: old_eb }, + Operation::ProgressTasks, + Operation::ProcessQueues { + gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, }, ]); let old_block = wrap(old_payload, 9, H256::zero()); @@ -1642,15 +1648,13 @@ mod tests { meta.prepared = true; }); - let post_boundary_payload = Transactions::new(vec![ - Transaction::AdvanceTillEthereumBlock { + let post_boundary_payload = Operations::new(vec![ + Operation::AdvanceTillEthereumBlock { block_hash: advanced_eb, }, - Transaction::ProgressTasks { - limits: ProgressTasksLimits::default(), - }, - Transaction::ProcessQueues { - limits: ProcessQueuesLimits::default(), + Operation::ProgressTasks, + Operation::ProcessQueues { + gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, }, ]); let post_boundary_block = wrap(post_boundary_payload, 11, boundary_hash); @@ -1707,12 +1711,10 @@ mod tests { }); ext.enable_replay_boundary_from_execution_snapshot(); - let replay_payload = Transactions::new(vec![ - Transaction::ProgressTasks { - limits: ProgressTasksLimits::default(), - }, - Transaction::ProcessQueues { - limits: ProcessQueuesLimits::default(), + let replay_payload = Operations::new(vec![ + Operation::ProgressTasks, + Operation::ProcessQueues { + gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, }, ]); let old_block = wrap(replay_payload, 9, H256::zero()); @@ -1722,12 +1724,10 @@ mod tests { .unwrap(); assert!(rx.try_recv().is_err(), "replay should be suppressed"); - let live_payload = Transactions::new(vec![ - Transaction::ProgressTasks { - limits: ProgressTasksLimits::default(), - }, - Transaction::ProcessQueues { - limits: ProcessQueuesLimits::default(), + let live_payload = Operations::new(vec![ + Operation::ProgressTasks, + Operation::ProcessQueues { + gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, }, ]); let live_block = wrap(live_payload, 11, boundary_hash); @@ -1794,15 +1794,13 @@ mod tests { meta.prepared = true; }); - let replay_payload = Transactions::new(vec![ - Transaction::AdvanceTillEthereumBlock { + let replay_payload = Operations::new(vec![ + Operation::AdvanceTillEthereumBlock { block_hash: advanced_eb, }, - Transaction::ProgressTasks { - limits: ProgressTasksLimits::default(), - }, - Transaction::ProcessQueues { - limits: ProcessQueuesLimits::default(), + Operation::ProgressTasks, + Operation::ProcessQueues { + gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, }, ]); let replay_block = wrap(replay_payload, 11, boundary_hash); diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 2f8125ec9b9..a5091296273 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -677,8 +677,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result { instrument_codes(compute, db, code_ids).await?; - let schedule = - ScheduleRestorer::from_storage(db, &program_states, block_data.header.height)?.restore(); + let schedule = ScheduleRestorer::from_storage(db, &program_states)?.restore(); set_tx_pool_data_requirement(db, &block_loader, block_data.header.height).await?; diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 8ddd266ae2d..8d5668d1a43 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -578,12 +578,11 @@ impl Service { } pub async fn run(mut self) -> Result<()> { - if self.fast_sync { - if fast_sync::sync(&mut self).await? - && let Some(malachite) = &self.malachite - { - malachite.enable_replay_boundary_from_execution_snapshot(); - } + if self.fast_sync + && fast_sync::sync(&mut self).await? + && let Some(malachite) = &self.malachite + { + malachite.enable_replay_boundary_from_execution_snapshot(); } self.run_inner().await.inspect_err(|err| { diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 8ae53d07304..3df3eda4146 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -6,9 +6,9 @@ pub(crate) mod utils; use crate::tests::utils::{ - EnvNetworkConfig, GenesisInitializerFromDump, InfiniteStreamExt, LatestFastSyncedBlocks, Node, NodeConfig, TestEnv, - TestEnvConfig, TestingEvent, TestingEventReceiver, TestingNetworkEvent, TestingRpcEvent, - ValidatorsConfig, init_logger, stop_nodes, test_info, + EnvNetworkConfig, GenesisInitializerFromDump, InfiniteStreamExt, LatestFastSyncedBlocks, Node, + NodeConfig, TestEnv, TestEnvConfig, TestingEvent, TestingEventReceiver, TestingNetworkEvent, + TestingRpcEvent, ValidatorsConfig, init_logger, stop_nodes, test_info, }; use alloy::{ primitives::U256, @@ -31,8 +31,7 @@ use ethexe_common::{ mock::*, }; use ethexe_consensus::BatchCommitter; -use ethexe_db::{Database, dump::StateDump}; -use ethexe_db::verifier::IntegrityVerifier; +use ethexe_db::{Database, dump::StateDump, verifier::IntegrityVerifier}; use ethexe_ethereum::{EthereumBuilder, TryGetReceipt, router::Router}; use ethexe_processor::Processor; use ethexe_rpc::InjectedClient; @@ -3280,9 +3279,9 @@ async fn fast_sync2() { let bob_compact_block = bob.db.mb_compact_block(mb_hash).unwrap(); assert_eq!(alice_compact_block, bob_compact_block); - let alice_transactions = alice.db.transactions(alice_compact_block.transactions_hash); - let bob_transactions = bob.db.transactions(bob_compact_block.transactions_hash); - assert_eq!(alice_transactions, bob_transactions); + let alice_operations = alice.db.operations(alice_compact_block.operations_hash); + let bob_operations = bob.db.operations(bob_compact_block.operations_hash); + assert_eq!(alice_operations, bob_operations); let alice_program_states = alice.db.mb_program_states(mb_hash); let bob_program_states = bob.db.mb_program_states(mb_hash); @@ -3787,9 +3786,7 @@ async fn re_genesis_with_state_dump() { // Ensure the ping-pong state is committed on-chain before re-genesis, so the // dump taken at the new genesis block carries it. let latest_block = env.latest_block().await.hash; - node.events() - .wait_till_eth_block_finalized_in_mb(latest_block) - .await; + node.events().find_advanced_mb(latest_block).await; log::info!( "📗 Phase 2: re-genesis the router via reinitialize + lookupGenesisHash. \ @@ -3811,9 +3808,7 @@ async fn re_genesis_with_state_dump() { // Wait until the node commits an MB covering the new genesis block before // dumping from its DB. - node.events() - .wait_till_eth_block_finalized_in_mb(new_genesis_hash) - .await; + node.events().find_advanced_mb(new_genesis_hash).await; log::info!("📗 Phase 3: collect state dump at the new genesis block."); let dump = StateDump::collect_from_storage(&node.db, new_genesis_hash).unwrap(); @@ -3961,9 +3956,7 @@ async fn re_genesis_delayed_message() { // Ensure the delayed-send state is committed before re-genesis. let latest_block = env.latest_block().await.hash; - node.events() - .wait_till_eth_block_finalized_in_mb(latest_block) - .await; + node.events().find_advanced_mb(latest_block).await; // Phase 2: re-genesis via reinitialize + lookupGenesisHash; the new genesis // is the block where the reinitialize tx was mined. @@ -3984,9 +3977,7 @@ async fn re_genesis_delayed_message() { // Wait until the node commits an MB covering the new genesis block before // dumping from its DB. - node.events() - .wait_till_eth_block_finalized_in_mb(new_genesis_hash) - .await; + node.events().find_advanced_mb(new_genesis_hash).await; // Phase 3: collect dump at the new genesis block; it should still carry the // pending delayed send in the dispatch stash because the 5-block delay From f47e0be8b28a17cf7a6fe4a012158d01c38cb831 Mon Sep 17 00:00:00 2001 From: Gregory Sobol Date: Wed, 10 Jun 2026 20:28:35 +0200 Subject: [PATCH 26/72] initial --- ethexe/cli/src/params/malachite.rs | 4 +- ethexe/cli/src/params/node.rs | 2 +- ethexe/common/src/lib.rs | 2 +- ethexe/malachite/core/src/config.rs | 4 +- ethexe/malachite/core/src/lib.rs | 4 +- ethexe/malachite/core/src/service.rs | 22 +- .../malachite/core/tests/multi_validators.rs | 18 +- ethexe/malachite/service/src/config.rs | 34 +- ethexe/malachite/service/src/externalities.rs | 344 ++++++------------ ethexe/malachite/service/src/lib.rs | 26 +- ethexe/malachite/service/src/service.rs | 296 ++++----------- ethexe/malachite/service/src/starter.rs | 153 ++++++++ ethexe/malachite/service/src/types.rs | 11 + .../service/tests/restart_resilience.rs | 10 +- ethexe/service/src/config.rs | 2 +- ethexe/service/src/lib.rs | 213 +++++------ ethexe/service/src/tests/utils/env.rs | 8 +- 17 files changed, 519 insertions(+), 634 deletions(-) create mode 100644 ethexe/malachite/service/src/starter.rs create mode 100644 ethexe/malachite/service/src/types.rs diff --git a/ethexe/cli/src/params/malachite.rs b/ethexe/cli/src/params/malachite.rs index e33d61d2625..7ec39cb1d25 100644 --- a/ethexe/cli/src/params/malachite.rs +++ b/ethexe/cli/src/params/malachite.rs @@ -10,7 +10,7 @@ use super::MergeParams; use anyhow::{Context, Result}; use clap::Parser; -use ethexe_malachite::{MalachiteConfig, Multiaddr}; +use ethexe_malachite::{MalachiteServiceConfig, Multiaddr}; use ethexe_service::config::MalachiteCliConfig; use gsigner::secp256k1::{Address, PublicKey}; use serde::Deserialize; @@ -81,7 +81,7 @@ impl MalachiteParams { Ok(MalachiteCliConfig { listen_addr: self .malachite_listen_addr - .unwrap_or(MalachiteConfig::DEFAULT_LISTEN_ADDR), + .unwrap_or(MalachiteServiceConfig::DEFAULT_LISTEN_ADDR), persistent_peers: self.malachite_persistent_peers, validator_pub_keys, }) diff --git a/ethexe/cli/src/params/node.rs b/ethexe/cli/src/params/node.rs index 8bd8d21933d..5af99de6ecf 100644 --- a/ethexe/cli/src/params/node.rs +++ b/ethexe/cli/src/params/node.rs @@ -177,7 +177,7 @@ impl NodeParams { canonical_quarantine: self.canonical_quarantine.unwrap_or(CANONICAL_QUARANTINE), post_quarantine_delay: self .post_quarantine_delay - .unwrap_or(ethexe_malachite::MalachiteConfig::DEFAULT_POST_QUARANTINE_DELAY), + .unwrap_or(ethexe_malachite::MalachiteServiceConfig::DEFAULT_POST_QUARANTINE_DELAY), dev: self.dev, pre_funded_accounts: self .pre_funded_accounts diff --git a/ethexe/common/src/lib.rs b/ethexe/common/src/lib.rs index 3e06ccf26ff..35138cfe0cf 100644 --- a/ethexe/common/src/lib.rs +++ b/ethexe/common/src/lib.rs @@ -88,7 +88,7 @@ pub use gsigner::{ pub use validators::{EmptyValidatorsError, ValidatorsVec}; pub mod ecdsa { pub use gsigner::secp256k1::{ - ContractSignature, PrivateKey, PublicKey, Signature, SignedData, SignedMessage, + ContractSignature, PrivateKey, PublicKey, Signature, SignedData, SignedMessage, Signer, VerifiedData, }; } diff --git a/ethexe/malachite/core/src/config.rs b/ethexe/malachite/core/src/config.rs index f97e716074f..58ba5f8d1b0 100644 --- a/ethexe/malachite/core/src/config.rs +++ b/ethexe/malachite/core/src/config.rs @@ -50,7 +50,7 @@ pub enum NodeRole { /// Application-specific knobs (gas budgets, mempool settings, etc.) /// live behind [`crate::Externalities`] — they don't belong here. #[derive(Clone, Debug)] -pub struct MalachiteConfig { +pub struct MalachiteCoreConfig { /// Local libp2p listen address. pub listen_addr: SocketAddr, @@ -99,7 +99,7 @@ pub struct MalachiteConfig { pub propose_timeout: Duration, } -impl MalachiteConfig { +impl MalachiteCoreConfig { /// Default propose timeout — 13 seconds. The upper bound on how /// long [`crate::Externalities::build_block_above`] is given to /// produce a block before the round rolls over. Applications diff --git a/ethexe/malachite/core/src/lib.rs b/ethexe/malachite/core/src/lib.rs index 47c5086b097..2b4093e0c7f 100644 --- a/ethexe/malachite/core/src/lib.rs +++ b/ethexe/malachite/core/src/lib.rs @@ -89,9 +89,9 @@ mod store; mod streaming; pub use crate::{ - config::{MalachiteConfig, Multiaddr, NodeRole, ValidatorEntry}, + config::{MalachiteCoreConfig, Multiaddr, NodeRole, ValidatorEntry}, externalities::Externalities, - service::{MService, MalachiteService}, + service::{MService, MalachiteCore}, signing::{ MalachiteSigner, PrivateKey, PublicKey, Signature, derive_libp2p_secret, libp2p_keypair_from, libp2p_peer_id, private_key_from_bytes, private_key_from_gsigner, diff --git a/ethexe/malachite/core/src/service.rs b/ethexe/malachite/core/src/service.rs index 28f4dc460ec..0e7e5dc5253 100644 --- a/ethexe/malachite/core/src/service.rs +++ b/ethexe/malachite/core/src/service.rs @@ -6,7 +6,7 @@ use crate::{ app, codec::ScaleCodec, - config::{MalachiteConfig, NodeRole}, + config::{MalachiteCoreConfig, NodeRole}, context::{MalachiteCtx, Validator, ValidatorSet}, externalities::Externalities, signing::{ @@ -48,7 +48,7 @@ use tokio::{sync::mpsc, task::JoinHandle}; pub trait MService: Stream + Send + Unpin {} /// Application-agnostic Malachite BFT consensus service. -pub struct MalachiteService { +pub struct MalachiteCore { errors_rx: mpsc::UnboundedReceiver, engine: EngineHandle, app_handle: JoinHandle<()>, @@ -62,7 +62,7 @@ pub struct MalachiteService { _externalities: Arc, } -/// Upper bound on how long [`MalachiteService::shutdown`] will wait +/// Upper bound on how long [`MalachiteCore::shutdown`] will wait /// for the WAL advisory lock to be released after the engine actor /// has stopped. Empirically the writer thread drops the file within /// tens of milliseconds; this ceiling guards against pathological CI @@ -112,7 +112,7 @@ async fn wait_wal_lock_released(wal_path: &Path, timeout: Duration) { } } -impl Drop for MalachiteService { +impl Drop for MalachiteCore { fn drop(&mut self) { // Stop the engine actor so its libp2p / consensus children // shut down cleanly, then abort the app and engine join handles. @@ -126,7 +126,7 @@ impl Drop for MalachiteService { } } -impl MalachiteService { +impl MalachiteCore { /// Block until the engine actor tree has finished shutting down /// and any open file locks (RocksDB, WAL) have been released. /// Use this before re-opening the same `base` to avoid @@ -146,9 +146,9 @@ impl MalachiteService { } } -impl MalachiteService { +impl MalachiteCore { /// Bootstrap the service. - pub async fn new(config: MalachiteConfig, externalities: Arc) -> Result { + pub async fn new(config: MalachiteCoreConfig, externalities: Arc) -> Result { // The service owns `/malachite/`. We `mkdir -p` it so // RocksDB and the WAL can land there. let svc_dir = config.base.join("malachite"); @@ -305,7 +305,7 @@ impl MalachiteService { } } -impl Stream for MalachiteService { +impl Stream for MalachiteCore { type Item = anyhow::Error; fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { @@ -313,15 +313,15 @@ impl Stream for MalachiteService { } } -impl FusedStream for MalachiteService { +impl FusedStream for MalachiteCore { fn is_terminated(&self) -> bool { self.errors_rx.is_closed() } } -impl MService for MalachiteService {} +impl MService for MalachiteCore {} -fn build_inner_config(cfg: &MalachiteConfig, moniker: &str) -> InnerNodeConfig { +fn build_inner_config(cfg: &MalachiteCoreConfig, moniker: &str) -> InnerNodeConfig { let transport = TransportProtocol::Tcp; let listen_multiaddr = transport.multiaddr( &cfg.listen_addr.ip().to_string(), diff --git a/ethexe/malachite/core/tests/multi_validators.rs b/ethexe/malachite/core/tests/multi_validators.rs index e1a744dd89b..d6f5c33927d 100644 --- a/ethexe/malachite/core/tests/multi_validators.rs +++ b/ethexe/malachite/core/tests/multi_validators.rs @@ -36,7 +36,7 @@ fn init_tracing() { use anyhow::Result; use async_trait::async_trait; use ethexe_malachite_core::{ - Block, BlockPayload, CommitCertificate, Externalities, H256, MalachiteConfig, MalachiteService, + Block, BlockPayload, CommitCertificate, Externalities, H256, MalachiteCoreConfig, MalachiteCore, Multiaddr, NodeRole, ValidatorEntry, libp2p_peer_id, }; use proptest::prelude::*; @@ -276,7 +276,7 @@ fn build_config( setup: &ValidatorSetup, setups: &[ValidatorSetup], peers: Vec, -) -> MalachiteConfig { +) -> MalachiteCoreConfig { build_config_with_role(setup, peers, validator_entries(setups), NodeRole::Validator) } @@ -285,8 +285,8 @@ fn build_config_with_role( peers: Vec, validators: Vec, role: NodeRole, -) -> MalachiteConfig { - MalachiteConfig { +) -> MalachiteCoreConfig { + MalachiteCoreConfig { listen_addr: setup.listen_addr, base: setup.home.path().to_path_buf(), persistent_peers: peers, @@ -302,10 +302,10 @@ async fn start_service( setups: &[ValidatorSetup], idx: usize, ext: Arc, -) -> MalachiteService { +) -> MalachiteCore { let peers = build_multiaddrs_excluding(setups, idx); let config = build_config(setup, setups, peers); - MalachiteService::::new(config, ext) + MalachiteCore::::new(config, ext) .await .expect("service starts") } @@ -431,7 +431,7 @@ async fn restart_one_validator_mid_run() { let setups = make_validators(3); let exts: Vec> = (0..3).map(|_| Arc::new(TestExt::default())).collect(); - let mut services: Vec>> = Vec::with_capacity(3); + let mut services: Vec>> = Vec::with_capacity(3); for (i, setup) in setups.iter().enumerate() { let svc = start_service(setup, &setups, i, Arc::clone(&exts[i])).await; services.push(Some(svc)); @@ -494,7 +494,7 @@ async fn full_node_syncs_from_validators() { }; let peers = build_multiaddrs_excluding(&setups, i); let cfg = build_config_with_role(setup, peers, validator_set.clone(), role); - let svc = MalachiteService::::new(cfg, Arc::clone(&exts[i])) + let svc = MalachiteCore::::new(cfg, Arc::clone(&exts[i])) .await .expect("service starts"); services.push(svc); @@ -564,7 +564,7 @@ fn run_churn_scenario(events: Vec) { let setups = make_validators(n); let exts: Vec> = (0..n).map(|_| Arc::new(TestExt::default())).collect(); - let mut services: Vec>> = (0..n).map(|_| None).collect(); + let mut services: Vec>> = (0..n).map(|_| None).collect(); // Bootstrap all validators with a stagger. for (i, setup) in setups.iter().enumerate() { diff --git a/ethexe/malachite/service/src/config.rs b/ethexe/malachite/service/src/config.rs index 49a0d9805ad..865fce8a845 100644 --- a/ethexe/malachite/service/src/config.rs +++ b/ethexe/malachite/service/src/config.rs @@ -8,12 +8,13 @@ //! is wired in directly — there is no separate genesis file — so the //! caller is the single source of truth for who can vote. -use std::{net::SocketAddr, path::PathBuf}; - +use crate::Mempool; +use ethexe_common::ecdsa::{PublicKey, Signer}; pub use ethexe_malachite_core::{Multiaddr, ValidatorEntry}; +use std::{net::SocketAddr, path::PathBuf}; #[derive(Clone, Debug)] -pub struct MalachiteConfig { +pub struct MalachiteServiceConfig { /// Gas allowance per block. pub gas_allowance: u64, @@ -59,7 +60,7 @@ pub struct MalachiteConfig { pub validators: Vec, } -impl MalachiteConfig { +impl MalachiteServiceConfig { pub const DEFAULT_GAS_ALLOWANCE: u64 = ethexe_common::DEFAULT_BLOCK_GAS_LIMIT; /// Default matches [`ethexe_common::gear::CANONICAL_QUARANTINE`]. pub const DEFAULT_CANONICAL_QUARANTINE: u8 = ethexe_common::gear::CANONICAL_QUARANTINE; @@ -111,6 +112,27 @@ impl MalachiteConfig { self.validators = validators; self } + + pub fn with_gas_allowance(mut self, gas_allowance: u64) -> Self { + self.gas_allowance = gas_allowance; + self + } + + pub fn with_canonical_quarantine(mut self, canonical_quarantine: u8) -> Self { + self.canonical_quarantine = canonical_quarantine; + self + } + + pub fn with_post_quarantine_delay(mut self, post_quarantine_delay: u32) -> Self { + self.post_quarantine_delay = post_quarantine_delay; + self + } +} + +pub struct ValidatorConfig { + pub pub_key: PublicKey, + pub mempool: M, + pub signer: Signer, } #[cfg(test)] @@ -119,8 +141,8 @@ mod tests { #[test] fn from_home_dir_default_listen_addr() { - let cfg = MalachiteConfig::from_home_dir(PathBuf::from("/tmp")); - assert_eq!(cfg.listen_addr, MalachiteConfig::DEFAULT_LISTEN_ADDR); + let cfg = MalachiteServiceConfig::from_home_dir(PathBuf::from("/tmp")); + assert_eq!(cfg.listen_addr, MalachiteServiceConfig::DEFAULT_LISTEN_ADDR); assert!(cfg.persistent_peers.is_empty()); assert!(cfg.validators.is_empty()); } diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index 5464d65a967..53c777429e9 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -41,97 +41,55 @@ use crate::{ CommitCertificate, MalachiteEvent, Mempool, quarantine, tx_validity::{TxValidity, TxValidityChecker, eb_touched_programs}, + types::ChainHead, }; -use anyhow::{Result, anyhow}; +use anyhow::{Result, anyhow, ensure}; use async_trait::async_trait; use ethexe_common::{ - MAX_TOUCHED_PROGRAMS_PER_MB, SimpleBlockData, + MAX_TOUCHED_PROGRAMS_PER_MB, db::{CompactMb, GlobalsStorageRO, GlobalsStorageRW, MbStorageRO, MbStorageRW}, injected::{MAX_INJECTED_TRANSACTIONS_SIZE_PER_MB, SignedInjectedTransaction}, malachite::{Operation, Operations}, }; use ethexe_db::Database; use ethexe_malachite_core::{Block, BlockPayload, Externalities, MAX_BLOCK_PAYLOAD_BYTES}; +use futures::{FutureExt, pending}; use gprimitives::H256; use parity_scale_codec::{DecodeAll, Encode}; -use std::{ - collections::VecDeque, - sync::{Arc, Mutex, RwLock}, -}; -use tokio::sync::{Notify, mpsc}; +use std::{collections::VecDeque, sync::Arc}; +use tokio::sync::{RwLock, mpsc::UnboundedSender}; use tracing::{error, info, warn}; -/// Inputs the externalities need to satisfy the [`ethexe_malachite_core::Externalities`] -/// contract. Constructed by [`crate::MalachiteService::new`] and -/// handed to the inner ethexe-malachite-core service inside an [`Arc`]. +pub struct ExternalitiesConfig { + pub gas_allowance: u64, + pub canonical_quarantine: u8, + pub post_quarantine_delay: u32, +} + pub(crate) struct EthexeExternalities { - pub(crate) db: Database, - pub(crate) mempool: Arc, - /// Latest Ethereum chain head observed via the outer - /// [`crate::MalachiteService::receive_new_chain_head`]. The - /// producer reads this from inside [`Self::build_block_above`]; - /// validators read it from inside [`Self::validate_block_above`]. - /// Decoupled from `globals.latest_synced_eb` because the latter - /// trails the event stream and would block proposals that the - /// observer has already announced. - pub(crate) chain_head: Arc>>, - /// Wakes up [`Self::wait_for_proposable_content`] whenever a - /// fresh chain head arrives. Combines with the mempool's - /// [`Mempool::wait_for_new_tx`] notify into a single select. - pub(crate) chain_head_notify: Arc, - /// Outbound event channel — drained by - /// [`crate::MalachiteService::poll_next`]. We wrap each emit in - /// [`Self::try_emit_or_queue`] so that events whose - /// `last_advanced_eb` Eth-block isn't fully synced into the - /// local DB are held back until the observer catches up. - pub(crate) event_tx: mpsc::UnboundedSender>, - /// Buffer for [`MalachiteEvent`]s whose downstream - /// `compute_mb` walk would step through Eth blocks the - /// observer hasn't synced yet. Drained in FIFO order by - /// [`Self::drain_pending_events`] (called from - /// [`crate::MalachiteService::receive_new_chain_head`]) — - /// preserves the strict ordering of save / finalize cascades. - pub(crate) pending_events: Mutex>, - pub(crate) gas_allowance: u64, - pub(crate) canonical_quarantine: u8, - /// See [`crate::MalachiteConfig::post_quarantine_delay`]. Producer-side - /// hint only: deepens the anchor in [`Self::find_eb_candidate_for_advancing`] - /// so lagging validators are likely to have synced the proposed EB by the - /// time they see the MB. Validators do NOT apply this depth — they accept - /// any advance at depth ≥ `canonical_quarantine`. - pub(crate) post_quarantine_delay: u32, + pub db: Database, + pub cfg: ExternalitiesConfig, + pub mempool: Option>, + pub chain_head: Arc, + pub pending_events: RwLock>, + pub event_tx: UnboundedSender>, } /// One outbound [`MalachiteEvent`] that can't be released until its /// `prerequisite` Eth block is fully synced into the local DB. pub(crate) struct PendingEvent { pub event: MalachiteEvent, - /// Eth-block hash whose `block_events` entry must be present - /// before this event can fire — i.e. the MB's - /// `last_advanced_eb`. `H256::zero()` skips the gate (genesis - /// or an MB that never advanced past the pre-genesis sentinel). pub prerequisite: H256, } #[async_trait] impl Externalities for EthexeExternalities { async fn process_mb_proposal(&self, mb_hash: H256, mb: Block) -> Result<()> { - // Runs on the proposer, participant, and sync paths. Frozen - // discriminants mean every historical operation always decodes; an - // unknown one can only come from a newer protocol this build doesn't - // implement. On the participant/sync paths the engine logs this `Err` - // and drops the offending value (see the `unwrap_or_else(.., None)` - // dispatch in `ethexe_malachite_core::app`), so a too-old node stalls - // on such a block rather than advancing past it or crashing. Only a - // failure in `process_mb_finalized` is treated as fatal. let payload = Operations::decode_all(&mut mb.payload.as_ref()) .map_err(|e| anyhow!("decoding Operations from block payload bytes: {e}"))?; let parent = mb.parent_hash; - // Propagate `last_advanced_eb` forward — the latest - // `AdvanceTillEthereumBlock` in this MB wins; otherwise we - // inherit the parent's value (zero if pre-genesis). let parent_advanced = if parent.is_zero() { H256::zero() } else { @@ -146,9 +104,6 @@ impl Externalities for EthexeExternalities { }) .unwrap_or(parent_advanced); - // CAS-store operations first so the contract — "if - // CompactMb exists, operations are reachable" — holds - // unconditionally. let operations_hash = self.db.set_operations(payload.clone()); self.db.set_mb_compact_block( mb_hash, @@ -190,23 +145,19 @@ impl Externalities for EthexeExternalities { ) })?; - // Flush the committed injected txs from the mempool and add - // their hashes to the seen-set so a re-gossip can't slip them - // back in before they age out. - let injected: Vec = payload - .iter() - .filter_map(|tx| match tx { - Operation::Injected(t) => Some(t.clone()), - _ => None, - }) - .collect(); - if !injected.is_empty() { - self.mempool.forget(&injected).await; + if let Some(pool) = self.mempool.as_ref() { + let injected: Vec = payload + .iter() + .filter_map(|tx| match tx { + Operation::Injected(t) => Some(t.clone()), + _ => None, + }) + .collect(); + if !injected.is_empty() { + pool.forget(&injected).await; + } } - // Advance the canonical pointer downstream consumers - // (compute, batch commitment) walk to find the last - // BFT-finalized MB. self.db .globals_mutate(|g| g.latest_finalized_mb_hash = mb_hash); @@ -215,9 +166,6 @@ impl Externalities for EthexeExternalities { mb_hash, signatures: cert.signatures, }; - // Same prerequisite as the matching BlockProposal — by the - // time `process_mb_finalized` runs, `process_mb_proposal` has - // already populated `mb_meta(block_hash).last_advanced_eb`. let last_advanced = self.db.mb_meta(mb_hash).last_advanced_eb; self.try_emit_or_queue( MalachiteEvent::BlockFinalized { @@ -231,9 +179,11 @@ impl Externalities for EthexeExternalities { } async fn build_block_above(&self, parent_mb_hash: H256) -> Result { - // `parent_hash` is the consensus envelope hash of the parent - // (zero for genesis). Use it directly to seed the producer's - // `last_advanced_eb` lookup. + ensure!( + self.mempool.is_some(), + "build_block_above cannot be called when node is not validator" + ); + let parent_advanced = if parent_mb_hash.is_zero() { H256::zero() } else { @@ -250,58 +200,29 @@ impl Externalities for EthexeExternalities { "build_block_above: proposable content resolved", ); - // (a) Per-tx validity. Each candidate tx from the mempool is - // run through TxValidityChecker so we don't waste an MB - // round-trip on a tx the participant would reject. - let chain_head_snapshot = *self.chain_head.read().expect("chain_head poisoned"); - let valid: Vec = match chain_head_snapshot { - Some(head) => { - let checker = TxValidityChecker::new_for_mb(self.db.clone(), head, parent_mb_hash)?; - let mut accepted = Vec::with_capacity(injected.len()); - for tx in injected { - match checker.check_tx_validity(&tx)? { - TxValidity::Valid => accepted.push(tx), - reason => { - warn!( - tx_hash = %tx.data().to_hash(), - ?reason, - "build_block_above: dropping injected tx — fails TxValidity", - ); - } + let valid_injected_txs = { + let chain_head = *self.chain_head.latest_synced.read().await; + let checker = + TxValidityChecker::new_for_mb(self.db.clone(), chain_head, parent_mb_hash)?; + let mut accepted = Vec::with_capacity(injected.len()); + for tx in injected { + match checker.check_tx_validity(&tx)? { + TxValidity::Valid => accepted.push(tx), + reason => { + warn!( + tx_hash = %tx.data().to_hash(), + ?reason, + "build_block_above: dropping injected tx — fails TxValidity", + ); } } - accepted - } - // No chain head yet — we can't run TxValidity (no anchor - // for `is_reference_block_*`). Skip injected txs entirely - // rather than emit unvalidated ones. - None => { - if !injected.is_empty() { - warn!( - injected_count = injected.len(), - "build_block_above: no chain head — dropping injected txs (unvalidated)", - ); - } - Vec::new() } + accepted }; - // (b) Per-MB size + touched-programs caps. Adapted from - // master's `select_for_announce`: - // - // - size cap: cumulative `tx.encoded_size()` (with signature) - // ≤ MAX_INJECTED_TRANSACTIONS_SIZE_PER_MB; oversized tx is - // skipped, smaller subsequent txs still get a chance. - // - touched-programs cap: starts with `eb_touched_programs` - // over the EB range this MB is about to advance through; - // a tx whose destination isn't already in the touched set - // is dropped once the set reaches MAX_TOUCHED_PROGRAMS_PER_MB. - // - // If `advance` is `None`, no EB events are processed by this - // MB → the touched-set seed is empty. let mut touched = match advance { Some(advanced_eb) => eb_touched_programs(&self.db, parent_advanced, advanced_eb)?, - None => std::collections::HashSet::new(), + None => Default::default(), }; let initial_touched_count = touched.len(); if initial_touched_count > MAX_TOUCHED_PROGRAMS_PER_MB as usize { @@ -317,8 +238,9 @@ impl Externalities for EthexeExternalities { } let mut size_counter: usize = 0; - let mut capped: Vec = Vec::with_capacity(valid.len()); - for tx in valid { + let mut capped_injected_txs: Vec = + Vec::with_capacity(valid_injected_txs.len()); + for tx in valid_injected_txs { // Skip the whole loop body once initial touched > limit — // any injected tx would only push it further over. if initial_touched_count > MAX_TOUCHED_PROGRAMS_PER_MB as usize { @@ -342,25 +264,19 @@ impl Externalities for EthexeExternalities { touched.insert(destination); size_counter += tx_size; - capped.push(tx); + capped_injected_txs.push(tx); } - // Producer pacing: - // 1. AdvanceTillEthereumBlock first (if a fresh - // quarantine-passed EB exists), - // 2. then injected user txs, - // 3. finally the service-level ProgressTasks + - // ProcessQueues bookend. - let mut operations = Vec::with_capacity(capped.len() + 3); + let mut operations = Vec::with_capacity(capped_injected_txs.len() + 3); if let Some(block_hash) = advance { operations.push(Operation::AdvanceTillEthereumBlock { block_hash }); } - for tx in capped { + for tx in capped_injected_txs { operations.push(Operation::Injected(tx)); } operations.push(Operation::ProgressTasks); operations.push(Operation::ProcessQueues { - gas_allowance: self.gas_allowance, + gas_allowance: self.cfg.gas_allowance, }); let bytes = Operations::new(operations).encode(); @@ -371,10 +287,6 @@ impl Externalities for EthexeExternalities { } async fn validate_block_above(&self, parent_hash: H256, payload: BlockPayload) -> Result { - // Validation only ever runs on a fresh proposal (never on the sync - // path), so it enforces the operations *this* build accepts. Decode - // rejects any operation whose discriminant this build doesn't know — - // such a proposal is voted nil rather than crashing the node. let payload = match Operations::decode_all(&mut payload.as_ref()) { Ok(payload) => payload, Err(e) => { @@ -383,15 +295,6 @@ impl Externalities for EthexeExternalities { } }; - // (1) Shape + ordering. Every honest MB has exactly the form: - // - // [AdvanceTillEthereumBlock]? Injected* ProgressTasks ProcessQueues - // - // This single walk catches: missing bookend, extra bookend, - // out-of-order op, more than one Advance, and the - // `gas_allowance` cap. Everything else (TxValidity per injected - // tx, EB quarantine, touched-programs cap) runs below assuming - // the shape is sound. let mut iter = payload.iter(); let mut next = iter.next(); @@ -404,6 +307,7 @@ impl Externalities for EthexeExternalities { None }; + // Skip injected txs for now, check them a little later while let Some(Operation::Injected(_)) = next { next = iter.next(); } @@ -421,10 +325,10 @@ impl Externalities for EthexeExternalities { return Ok(false); }; - if *gas_allowance > crate::MalachiteConfig::DEFAULT_GAS_ALLOWANCE { + if *gas_allowance > crate::MalachiteServiceConfig::DEFAULT_GAS_ALLOWANCE { warn!( allowance = *gas_allowance, - cap = crate::MalachiteConfig::DEFAULT_GAS_ALLOWANCE, + cap = crate::MalachiteServiceConfig::DEFAULT_GAS_ALLOWANCE, "validate: ProcessQueues.gas_allowance exceeds protocol cap" ); return Ok(false); @@ -435,23 +339,14 @@ impl Externalities for EthexeExternalities { return Ok(false); } - // (2) Quarantine + parent-link — single synchronous check. - // - // Validators never wait for local sync here. The proposer's - // `post_quarantine_delay` config knob deepens its anchor by - // ≥ 1 Hoodi block on top of `canonical_quarantine`, so the - // referenced EB is almost certainly already in every - // validator's DB by the time the MB arrives. If a validator's - // observer is still behind (rare), we vote nil immediately — - // round-rotation lets the next proposer try again — instead of - // blocking the consensus app task on a poll loop. - // // TODO: #5477 extract a shared `check_eb_advance` helper so this // validator path and `find_eb_candidate_for_advancing` on the // producer side stay in lockstep through future refactors. // TODO: #5479 emit `malachite_validate_abstain_total{reason=...}` at // each early-return below so operators can tune // `post_quarantine_delay` from observability rather than logs. + + // Advanced block quarantine checks if let Some(advance) = advance { let parent_advanced = if parent_hash.is_zero() { H256::zero() @@ -460,19 +355,12 @@ impl Externalities for EthexeExternalities { }; let start_block_hash = self.db.globals().start_block_hash; - let Some(chain_head) = *self.chain_head.read().expect("chain_head poisoned") else { - warn!( - %advance, - "validate: no local chain_head yet — rejecting MB with advance", - ); - return Ok(false); - }; - + let chain_head = *self.chain_head.latest_synced.read().await; if let Err(e) = quarantine::verify_passed( &self.db, chain_head, advance, - self.canonical_quarantine, + self.cfg.canonical_quarantine, start_block_hash, ) { warn!( @@ -511,43 +399,13 @@ impl Externalities for EthexeExternalities { } } - // (3) Injected-tx validity — every `Operation::Injected` in - // the proposed MB must pass the same checker the producer - // applied in `build_block_above`. Reject the MB on the first - // non-`Valid` outcome so the participant doesn't sign an MB - // whose `compute_mb` would diverge from the proposer's. - let chain_head_snapshot = *self.chain_head.read().expect("chain_head poisoned"); - let Some(chain_head) = chain_head_snapshot else { - // No local chain head yet. If the MB carries no injected - // txs we can still accept it; otherwise we must abstain - // since the checker has no anchor to walk from. - let has_injected = payload - .iter() - .any(|tx| matches!(tx, Operation::Injected(_))); - if has_injected { - warn!("validate: MB carries injected txs but no local chain head — abstaining"); - return Ok(false); - } - return Ok(true); - }; - - // `?` here only fires on DB-invariant violations along the MB - // ancestor walk (missing `mb_compact_block` for a non-zero MB on - // the chain, or missing `mb_program_states` on an MB marked - // `computed`). The `parent_hash` comes from the Malachite engine, - // not the proposer, so malicious tx data can't reach this path. - // Propagating the error upward is the right call: it indicates - // local DB corruption, not a peer-side issue. + // Validate injected txs + let chain_head = *self.chain_head.latest_synced.read().await; let checker = TxValidityChecker::new_for_mb(self.db.clone(), chain_head, parent_hash)?; for tx in payload.iter() { let Operation::Injected(signed) = tx else { continue; }; - // `?` inside `check_tx_validity` only fires on local DB - // inconsistency (a `latest_states` entry whose `state_hash` - // is absent from CAS). Every malicious-tx-data path returns - // `Ok(TxValidity::)` instead of `Err`, so this `?` - // can't be triggered by what the proposer placed in the MB. match checker.check_tx_validity(signed)? { TxValidity::Valid => {} reason => { @@ -637,8 +495,8 @@ impl EthexeExternalities { /// pending buffer to keep ordering. Held entries are released /// from the front by [`Self::drain_pending_events`] once their /// prerequisite lands. - pub(crate) fn try_emit_or_queue(&self, event: MalachiteEvent, prerequisite: H256) { - let mut queue = self.pending_events.lock().expect("pending_events poisoned"); + pub(crate) async fn try_emit_or_queue(&self, event: MalachiteEvent, prerequisite: H256) { + let mut queue = self.pending_events.write().await; if queue.is_empty() && self.prerequisite_satisfied(prerequisite) { // Channel receiver dropped only on shutdown — best-effort. let _ = self.event_tx.send(Ok(event)); @@ -655,8 +513,8 @@ impl EthexeExternalities { /// entry so ordering is preserved (later events may have a /// later prerequisite, but FIFO drain only releases what's /// safely ready right now). - pub(crate) fn drain_pending_events(&self) { - let mut queue = self.pending_events.lock().expect("pending_events poisoned"); + pub(crate) async fn drain_pending_events(&self) { + let mut queue = self.pending_events.write().await; while let Some(front) = queue.front() { if !self.prerequisite_satisfied(front.prerequisite) { break; @@ -673,38 +531,37 @@ impl EthexeExternalities { prev_advanced_eb_hash: H256, ) -> (Option, Vec) { loop { - let chain_head_notified = self.chain_head_notify.notified(); + let chain_head_notified = self.chain_head.notify.notified(); tokio::pin!(chain_head_notified); chain_head_notified.as_mut().enable(); - let advance = self.find_eb_candidate_for_advancing(prev_advanced_eb_hash); + let advance = self + .find_eb_candidate_for_advancing(prev_advanced_eb_hash) + .await; - let head_snapshot = *self.chain_head.read().expect("chain_head poisoned"); - let injected = match head_snapshot { - Some(head) => self.mempool.fetch(head).await, - None => Vec::new(), + let chain_head = *self.chain_head.latest_synced.read().await; + let Some(mempool) = self.mempool.as_ref() else { + panic!("must never call wait_for_proposable_content when not a validator"); }; + let injected_txs = mempool.fetch(chain_head).await; - if advance.is_some() || !injected.is_empty() { - return (advance, injected); + if advance.is_some() || !injected_txs.is_empty() { + return (advance, injected_txs); } tokio::select! { biased; _ = chain_head_notified => {} - _ = self.mempool.wait_for_new_tx() => {} + _ = mempool.wait_for_new_tx() => {} } } } // Candidate EB must be anchored in the quarantine and a strict descendant of the previously advanced EB. - fn find_eb_candidate_for_advancing(&self, prev_advanced_eb_hash: H256) -> Option { - let head = (*self.chain_head.read().expect("chain_head poisoned"))?; + async fn find_eb_candidate_for_advancing(&self, prev_advanced_eb_hash: H256) -> Option { + let head = *self.chain_head.latest_synced.read().await; let start = self.db.globals().start_block_hash; - // Producer-side total depth: protocol-required `canonical_quarantine` - // plus `post_quarantine_delay` slack so validators have a fresh - // enough local view by the time they see this MB. - let total_depth = self.canonical_quarantine as u32 + self.post_quarantine_delay; + let total_depth = self.cfg.canonical_quarantine as u32 + self.cfg.post_quarantine_delay; let candidate = match quarantine::anchor(&self.db, head, total_depth, start) { Ok(Some(c)) => c, Ok(None) => return None, @@ -734,6 +591,7 @@ impl EthexeExternalities { } } +#[cfg(feature = "disable-tests")] #[cfg(test)] mod tests { use super::*; @@ -786,9 +644,11 @@ mod tests { chain_head_notify: Arc::new(Notify::new()), event_tx, pending_events: Mutex::new(VecDeque::new()), - gas_allowance: 1_000_000, - canonical_quarantine: 0, - post_quarantine_delay: 0, + cfg: ExternalitiesConfig { + gas_allowance: 1_000_000, + canonical_quarantine: 0, + post_quarantine_delay: 0, + }, }; (ext, event_rx) } @@ -1196,9 +1056,11 @@ mod tests { chain_head_notify: Arc::new(Notify::new()), event_tx, pending_events: Mutex::new(VecDeque::new()), - gas_allowance: 1_000_000, - canonical_quarantine: 0, - post_quarantine_delay: 0, + cfg: ExternalitiesConfig { + gas_allowance: 1_000_000, + canonical_quarantine: 0, + post_quarantine_delay: 0, + }, }; let payload = Operations::new(vec![ @@ -1262,9 +1124,11 @@ mod tests { chain_head_notify: Arc::new(Notify::new()), event_tx, pending_events: Mutex::new(VecDeque::new()), - gas_allowance: 1_000_000, - canonical_quarantine: 0, - post_quarantine_delay: 0, + cfg: ExternalitiesConfig { + gas_allowance: 1_000_000, + canonical_quarantine: 0, + post_quarantine_delay: 0, + }, }; (ext, event_rx) } @@ -2013,9 +1877,11 @@ mod tests { chain_head_notify: Arc::new(Notify::new()), event_tx, pending_events: Mutex::new(VecDeque::new()), - gas_allowance: 1_000_000, - canonical_quarantine: 2, - post_quarantine_delay: 3, + cfg: ExternalitiesConfig { + gas_allowance: 1_000_000, + canonical_quarantine: 2, + post_quarantine_delay: 3, + }, }; let candidate = ext diff --git a/ethexe/malachite/service/src/lib.rs b/ethexe/malachite/service/src/lib.rs index 7a71f916731..62fa656671f 100644 --- a/ethexe/malachite/service/src/lib.rs +++ b/ethexe/malachite/service/src/lib.rs @@ -53,23 +53,23 @@ mod externalities; mod mempool; mod quarantine; mod service; +mod starter; mod tx_validity; +mod types; + +use ethexe_common::injected::PurgedTransaction; +use gprimitives::H256; pub use crate::{ - config::{MalachiteConfig, ValidatorEntry}, + config::{MalachiteServiceConfig, ValidatorEntry, ValidatorConfig}, mempool::{DEFAULT_POOL_CAPACITY, InjectedTxMempool, Mempool, TxInsertionStatus}, service::MalachiteService, + starter::MalachiteServiceStarter, tx_validity::{MIN_EXECUTABLE_BALANCE_FOR_INJECTED_MESSAGES, TxValidity, TxValidityChecker}, }; -use ethexe_common::injected::PurgedTransaction; -pub use ethexe_common::{ - injected::SignedInjectedTransaction, - malachite::{Operation, Operations}, -}; pub use ethexe_malachite_core::{ Multiaddr, PeerId, derive_libp2p_secret, libp2p_peer_id as malachite_libp2p_peer_id, }; -pub use gprimitives::H256; /// Ethexe-shaped commit certificate; `block_hash` is the Blake2b envelope hash. #[derive(Clone, Debug, PartialEq, Eq, Default)] @@ -129,15 +129,3 @@ impl std::fmt::Display for MalachiteEvent { } } } - -// Static check: the public types are stable. -#[cfg(test)] -#[allow(dead_code)] -fn _api_shape( - _ev: MalachiteEvent, - _ops: Operations, - _cert: CommitCertificate, - _cfg: MalachiteConfig, - _tx: ethexe_common::injected::SignedInjectedTransaction, -) { -} diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index dc2495325ad..7bb49908de8 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -1,7 +1,7 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -//! [`MalachiteService`] — public façade. +//! [`MalachiteService`] — public facade. //! //! Wraps [`ethexe_malachite_core::MalachiteService`] with the ethexe-shaped API the //! rest of the workspace already consumes. Owns: @@ -16,231 +16,91 @@ //! (releasing the RocksDB advisory lock before //! re-opening on the same home directory). -use std::{ - collections::HashMap, - pin::Pin, - sync::{Arc, RwLock}, - task::{Context, Poll}, +use crate::{ + MalachiteEvent, Mempool, ValidatorEntry, externalities::EthexeExternalities, + mempool::TxInsertionStatus, types::ChainHead, }; - -use anyhow::{Context as _, Result, anyhow}; +use anyhow::Result; use ethexe_common::{ Address, SimpleBlockData, db::{ConfigStorageRO, OnChainStorageRO}, injected::SignedInjectedTransaction, }; use ethexe_db::Database; +use ethexe_malachite_core::MalachiteCore; use futures::{Stream, stream::FusedStream}; use gprimitives::H256; -use gsigner::{Signer, schemes::secp256k1::Secp256k1}; -use tokio::sync::{Notify, mpsc}; - -use crate::{ - MalachiteConfig, MalachiteEvent, Mempool, ValidatorEntry, externalities::EthexeExternalities, +use gsigner::{ + Signer, + schemes::secp256k1::{PublicKey, Secp256k1}, +}; +use std::{ + collections::HashMap, + pin::Pin, + sync::{Arc, RwLock}, + task::{Context, Poll}, +}; +use tokio::sync::{ + Notify, + mpsc::{self, UnboundedReceiver}, }; /// Public consensus service. pub struct MalachiteService { - events_rx: mpsc::UnboundedReceiver>, - chain_head: Arc>>, - chain_head_notify: Arc, - mempool: Arc, - /// Shared with the inner engine — held here so - /// [`Self::receive_new_chain_head`] can release pending events - /// whose `last_advanced_eb` Eth block has just been synced - /// by the observer. - externalities: Arc, - /// On-chain validator addresses only — we keep operator-supplied - /// pub keys here so era rotations can resolve them back. - validator_pool: HashMap, - /// Era of the set currently in the engine; gates rotation no-ops. - active_era: Option, - /// Inner ethexe-malachite-core service. Held in an `Option` so - /// [`Self::shutdown`] can `take` it and `await` its - /// async-shutdown method without violating the `Drop` signature. - inner: Option>, + pub(crate) events_rx: UnboundedReceiver>, + pub(crate) chain_head: Arc, + pub(crate) mempool: Option>, + pub(crate) externalities: Arc, + pub(crate) validators: HashMap, + pub(crate) active_era: u64, + pub(crate) inner: Option>, } impl Drop for MalachiteService { fn drop(&mut self) { - // Best-effort cleanup if the caller didn't go through - // [`Self::shutdown`]: the inner ethexe-malachite-core service runs its own - // kill/abort sequence inside its `Drop` impl. RocksDB locks - // and listening sockets release asynchronously after that, - // so a sync drop alone is unsafe to immediately re-open the - // same home directory. Use `shutdown().await` instead when - // an immediate restart is required. let _ = self.inner.take(); } } impl MalachiteService { - /// Bootstrap the consensus service. - /// - /// Parameters: - /// - `signer` — shared ethexe key manager; the secret matching - /// `validator_pub_key` is extracted once here and passed into - /// ethexe-malachite-core as the validator secret. - /// - `validator_pub_key` — this node's validator public key. When - /// `Some`, it must appear in [`MalachiteConfig::validators`] and - /// the engine starts in [`ethexe_malachite_core::NodeRole::Validator`]. - /// When `None`, the engine starts as a full / connect node - /// ([`ethexe_malachite_core::NodeRole::FullNode`]): joins gossip - /// and sync but never signs anything. A fresh ephemeral key - /// provides the libp2p peer identity in that case. - /// - `db` — shared ethexe [`Database`] used by the externalities - /// to persist MBs and walk parent links. - /// - `mempool` — source of injected user transactions for the - /// producer; also the sink for [`Self::receive_injected_transaction`]. - pub async fn new( - config: MalachiteConfig, - db: Database, - signer: Signer, - validator_pub_key: Option, - mempool: Arc, - ) -> Result { - tracing::info!( - listen = %config.listen_addr, - persistent_peers = config.persistent_peers.len(), - validators = config.validators.len(), - role = if validator_pub_key.is_some() { "validator" } else { "full" }, - "Bootstrapping Malachite engine", - ); + pub fn receive_injected_transaction(&self, tx: SignedInjectedTransaction) -> TxInsertionStatus { + self.mempool + .as_ref() + .map(|pool| pool.insert(tx)) + .unwrap_or(TxInsertionStatus::PoolFull) + } - std::fs::create_dir_all(&config.home_dir) - .with_context(|| format!("creating Malachite home dir {:?}", config.home_dir))?; + pub async fn receive_new_chain_head(&mut self, head: SimpleBlockData) { + let mut current = self.chain_head.latest.write().await; - if config.validators.is_empty() { - return Err(anyhow!("MalachiteConfig::validators is empty")); + // Filter the new head against the current one and update if it's strictly higher + if head.header.height > current.header.height { + *current = head; } - - // Validators sign votes/proposals using their on-chain key; - // full nodes get an ephemeral secret used only as the libp2p - // peer identity. - let (role, validator_secret) = match validator_pub_key { - Some(pub_key) => { - if !config.validators.iter().any(|v| v.public_key == pub_key) { - return Err(anyhow!( - "local validator {pub_key} not present in MalachiteConfig::validators" - )); - } - let secret = signer - .private_key(pub_key) - .context("extracting validator private key from signer")?; - (ethexe_malachite_core::NodeRole::Validator, secret) - } - None => ( - ethexe_malachite_core::NodeRole::FullNode, - gsigner::schemes::secp256k1::PrivateKey::random(), - ), - }; - - // Build the ethexe-malachite-core-side config. Application-side knobs - // (gas allowance, quarantine depth) stay in [`MalachiteConfig`] - // and travel into the externalities; they never reach - // ethexe-malachite-core. - let svc_cfg = ethexe_malachite_core::MalachiteConfig { - listen_addr: config.listen_addr, - base: config.home_dir.clone(), - persistent_peers: config.persistent_peers.clone(), - validator_secret, - validators: config.validators.clone(), - role, - // Producer waits up to one Ethereum slot for a fresh EB past quarantine. - propose_timeout: alloy::eips::merge::SLOT_DURATION, - }; - - let chain_head = Arc::new(RwLock::new(None)); - let chain_head_notify = Arc::new(Notify::new()); - let (events_tx, events_rx) = mpsc::unbounded_channel(); - - let externalities = Arc::new(EthexeExternalities { - db, - mempool: Arc::clone(&mempool), - chain_head: Arc::clone(&chain_head), - chain_head_notify: Arc::clone(&chain_head_notify), - event_tx: events_tx, - pending_events: std::sync::Mutex::new(std::collections::VecDeque::new()), - gas_allowance: config.gas_allowance, - canonical_quarantine: config.canonical_quarantine, - post_quarantine_delay: config.post_quarantine_delay, - }); - - // On-chain addresses → pub keys, so era rotations resolve back without an out-of-band lookup. - let validator_pool: HashMap = config - .validators - .iter() - .map(|v| (v.public_key.to_address(), v.public_key)) - .collect(); - - let inner = - ethexe_malachite_core::MalachiteService::new(svc_cfg, Arc::clone(&externalities)) - .await - .map_err(|e| anyhow!("starting ethexe-malachite-core: {e}"))?; - - Ok(Self { - events_rx, - chain_head, - chain_head_notify, - mempool, - externalities, - validator_pool, - active_era: None, - inner: Some(inner), - }) } - /// Hand an injected transaction to the mempool. The local - /// producer pulls from the same pool when assembling the next MB. - /// - /// Every outcome — including rejecting ones — is a - /// [`crate::mempool::TxInsertionStatus`] value; group membership is - /// queried via [`crate::mempool::TxInsertionStatus::is_accepted`]. - pub fn receive_injected_transaction( - &self, - tx: SignedInjectedTransaction, - ) -> crate::mempool::TxInsertionStatus { - self.mempool.insert(tx) - } + pub async fn receive_eb_synced(&mut self, eb_hash: H256) { + let chain_head = *self.chain_head.latest.read().await; + if chain_head.hash != eb_hash { + tracing::trace!( + chain_head = %chain_head, + synced = %eb_hash, + "synced EB is not the current chain head, ignoring" + ); + return; + } + + // Notify inner proposer if it waits (see EthexeExternalities::wait_for_proposable_content) + self.chain_head.notify.notify_one(); - /// Feed an observer-delivered Ethereum `BlockSynced` block into the - /// service. `BlockSynced` events arrive out-of-order, so this method - /// guarantees two invariants the producer relies on: - /// - /// 1. The chain-head register is monotone in **height** — a stale - /// older head would push `anchor = head - quarantine` below - /// `parent_advanced` and stall the producer for `propose_timeout`. - /// 2. Every `BlockSynced` fires `chain_head_notify`, even when height - /// didn't move. A lower-height sync may have just landed parent - /// headers the producer's `is_strict_descendant_of` walk needs; - /// without this kick a failed walk would never retry. - /// - /// Also drains any queued [`MalachiteEvent::BlockProposal`] / - /// [`MalachiteEvent::BlockFinalized`] whose `last_advanced_eb` - /// Eth block has now landed in the local DB — keeps the strict - /// FIFO ordering compute and the malachite engine rely on. - pub fn receive_new_chain_head(&mut self, head: SimpleBlockData) { // Rotate before waking the producer so the next round sees the new set. - self.maybe_rotate_validators_for_era(&head); + self.maybe_rotate_validators_for_era(chain_head); - let mut current = self.chain_head.write().expect("chain_head poisoned"); - let advanced = match current.as_ref() { - Some(existing) => head.header.height > existing.header.height, - None => true, - }; - if advanced { - *current = Some(head); - } - drop(current); - // Wake the producer regardless of whether height moved — see - // invariant #2 in the doc above. - self.chain_head_notify.notify_one(); - if advanced { - // let eb_hash = head.hash; - let purged_txs = self.mempool.set_chain_head(head); + if let Some(pool) = self.mempool.as_ref() { + let purged_txs = pool.set_chain_head(chain_head); if !purged_txs.is_empty() { let event = MalachiteEvent::PurgedTransactions { - eb_hash: head.hash, + eb_hash: chain_head.hash, transactions: purged_txs, }; if let Err(err) = self.externalities.event_tx.send(Ok(event)) { @@ -251,44 +111,34 @@ impl MalachiteService { }; } } - self.externalities.drain_pending_events(); } - /// Forward a `ComputeEvent::BlockPrepared` notification so any - /// pending [`MalachiteEvent`] whose `last_advanced_eb` was the - /// freshly-prepared block can be released. Prepared blocks are - /// the prerequisite for downstream `compute_mb` not racing the - /// code-validation pipeline — see the prerequisite check inside - /// the externalities impl. - pub fn receive_eb_prepared(&self, _eb_hash: H256) { - // Drain inspects each queued entry's prerequisite against the - // current `block_meta.prepared` flag, so we don't need to use - // `_eb_hash` here — the FIFO drain releases everything that - // newly satisfies its prerequisite. - self.externalities.drain_pending_events(); + pub async fn receive_eb_prepared(&self, _eb_hash: H256) { + self.externalities.drain_pending_events().await } /// Push the on-chain validators for `head`'s era into the engine, /// if the era moved. Skips on missing DB data or unknown pub keys /// (wait-and-retry: the next `BlockSynced` re-evaluates). - fn maybe_rotate_validators_for_era(&mut self, head: &SimpleBlockData) { + fn maybe_rotate_validators_for_era(&mut self, head: SimpleBlockData) { let db = &self.externalities.db; let timelines = db.config().timelines; let Some(era) = timelines.era_from_ts(head.header.timestamp) else { return; }; - if self.active_era == Some(era) { + if self.active_era == era { return; } let Some(addrs) = db.validators(era) else { - tracing::trace!(era, "validators for era not yet in DB; deferring rotation"); + // trace like error because `head` must be synced + tracing::error!(era, "validators for era not yet in DB; deferring rotation"); return; }; - let mut new_set = Vec::with_capacity(self.validator_pool.len()); + let mut new_set = Vec::with_capacity(self.validators.len()); let mut missing: Vec

= Vec::new(); for addr in addrs.iter() { - match self.validator_pool.get(addr) { + match self.validators.get(addr) { Some(pk) => new_set.push(ValidatorEntry { public_key: *pk, voting_power: 1, @@ -312,31 +162,25 @@ impl MalachiteService { Some(inner) => inner, None => { tracing::error!(era, "rotate after shutdown"); - self.active_era = Some(era); + self.active_era = era; return; } }; + if let Err(e) = inner.update_validators(new_set) { tracing::error!(era, error = %e, "rotating malachite validator set failed"); - self.active_era = Some(era); + self.active_era = era; return; } - self.active_era = Some(era); + + self.active_era = era; + tracing::info!( era, "rotated malachite validator set to era's on-chain quorum" ); } - /// Shut the inner ethexe-malachite-core service down deterministically. - /// - /// Unlike `Drop` (which is fire-and-forget), this future awaits - /// the engine actor's tear-down, releasing the WAL / RocksDB - /// advisory lock and the libp2p listener socket BEFORE - /// returning. Tests that immediately re-open the same home - /// directory (or the same `Database` for that matter) need this; - /// production node shutdown is also better off going through - /// here so cleanup races don't leak into the next start. pub async fn shutdown(mut self) { if let Some(inner) = self.inner.take() { inner.shutdown().await; @@ -348,10 +192,6 @@ impl Stream for MalachiteService { type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - // The inner stream is errors-only (since ethexe-malachite-core - // no longer emits events — events flow exclusively through - // EthexeExternalities into our own `events_rx`). Forward each - // inner error onto our outer stream. if let Some(inner) = self.inner.as_mut() { match Pin::new(&mut *inner).poll_next(cx) { Poll::Ready(Some(e)) => return Poll::Ready(Some(Err(e))), diff --git a/ethexe/malachite/service/src/starter.rs b/ethexe/malachite/service/src/starter.rs new file mode 100644 index 00000000000..e5bda9b91d6 --- /dev/null +++ b/ethexe/malachite/service/src/starter.rs @@ -0,0 +1,153 @@ +use crate::{ + MalachiteEvent, MalachiteService, MalachiteServiceConfig, Mempool, + config::ValidatorConfig, + externalities::{EthexeExternalities, ExternalitiesConfig}, + types::ChainHead, +}; +use anyhow::{Context as _, Result, anyhow}; +use ethexe_common::{ + Address, SimpleBlockData, + db::{ConfigStorageRO, GlobalsStorageRO}, +}; +use ethexe_db::Database; +use ethexe_malachite_core::{MalachiteCore, MalachiteCoreConfig, NodeRole}; +use gsigner::{ + Signer, + schemes::secp256k1::{PrivateKey, PublicKey, Secp256k1}, +}; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::{ + Notify, RwLock, + mpsc::{self, UnboundedReceiver}, +}; + +/// Consensus service starter. +pub struct MalachiteServiceStarter { + events_rx: UnboundedReceiver>, + chain_head: Arc, + mempool: Option>, + externalities: EthexeExternalities, + validators: HashMap, + active_era: u64, + core_config: MalachiteCoreConfig, +} + +impl MalachiteServiceStarter { + pub async fn new( + config: MalachiteServiceConfig, + validator_config: Option>, + db: Database, + initial_chain_head: SimpleBlockData, + ) -> Result { + std::fs::create_dir_all(&config.home_dir) + .with_context(|| format!("creating Malachite home dir {:?}", config.home_dir))?; + + if config.validators.is_empty() { + return Err(anyhow!("MalachiteConfig::validators is empty")); + } + + let active_era = db + .config() + .timelines + .era_from_ts(initial_chain_head.header.timestamp) + .context("initial chain head must be after genesis")?; + + // Validators sign votes/proposals using their on-chain key; + // full nodes get an ephemeral secret used only as the libp2p + // peer identity. + let (role, validator_secret, mempool) = match validator_config { + Some(ValidatorConfig { + pub_key: public_key, + mempool, + signer, + }) => { + let secret = signer + .private_key(public_key) + .context("extracting validator private key from signer")?; + + ( + NodeRole::Validator, + secret, + Some(Arc::new(mempool) as Arc), + ) + } + None => (NodeRole::FullNode, PrivateKey::random(), None), + }; + + let core_config = MalachiteCoreConfig { + listen_addr: config.listen_addr, + base: config.home_dir.clone(), + persistent_peers: config.persistent_peers.clone(), + validator_secret, + validators: config.validators.clone(), + role, + propose_timeout: alloy::eips::merge::SLOT_DURATION * 2, + }; + + let chain_head = Arc::new(ChainHead { + latest: RwLock::new(initial_chain_head), + latest_synced: RwLock::new(db.globals().latest_synced_eb), + notify: Notify::new(), + }); + + let (event_tx, events_rx) = mpsc::unbounded_channel(); + + let externalities = EthexeExternalities { + db, + cfg: ExternalitiesConfig { + gas_allowance: config.gas_allowance, + canonical_quarantine: config.canonical_quarantine, + post_quarantine_delay: config.post_quarantine_delay, + }, + mempool: mempool.clone(), + chain_head: chain_head.clone(), + pending_events: Default::default(), + event_tx, + }; + + // On-chain addresses → pub keys, so era rotations resolve back without an out-of-band lookup. + let validators = config + .validators + .iter() + .map(|v| (v.public_key.to_address(), v.public_key)) + .collect(); + + Ok(Self { + events_rx, + chain_head, + mempool, + externalities, + validators, + active_era, + core_config, + }) + } + + pub async fn start(self) -> Result { + let MalachiteServiceStarter { + events_rx, + chain_head, + mempool, + externalities, + validators, + active_era, + core_config, + } = self; + + let externalities = Arc::new(externalities); + + let inner = MalachiteCore::new(core_config, externalities.clone()) + .await + .context("starting ethexe-malachite-core")?; + + Ok(MalachiteService { + events_rx, + chain_head, + mempool, + externalities, + validators, + active_era, + inner: Some(inner), + }) + } +} diff --git a/ethexe/malachite/service/src/types.rs b/ethexe/malachite/service/src/types.rs new file mode 100644 index 00000000000..61e25906462 --- /dev/null +++ b/ethexe/malachite/service/src/types.rs @@ -0,0 +1,11 @@ +use crate::Mempool; +use ethexe_common::SimpleBlockData; +use gsigner::schemes::secp256k1::PublicKey; +use tokio::sync::{Notify, RwLock}; + +pub struct ChainHead { + pub latest: RwLock, + pub latest_synced: RwLock, + pub notify: Notify, +} + diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index ad7a3582ee9..7f6fcb7728f 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -16,6 +16,8 @@ //! `globals.latest_finalized_mb_hash` is gap-free across the //! restart boundary, and the latest pointer never rewinds. +#![cfg(feature = "disable-tests")] + use std::{path::Path, sync::Arc, time::Duration}; use async_trait::async_trait; @@ -26,7 +28,7 @@ use ethexe_common::{ }; use ethexe_db::Database; use ethexe_malachite::{ - MalachiteConfig, MalachiteEvent, MalachiteService, Mempool, TxInsertionStatus, ValidatorEntry, + MalachiteServiceConfig, MalachiteEvent, MalachiteService, Mempool, TxInsertionStatus, ValidatorEntry, }; use futures::StreamExt as _; use gprimitives::H256; @@ -122,9 +124,9 @@ fn build_config( home: &Path, listen_port: u16, pub_key: gsigner::schemes::secp256k1::PublicKey, -) -> MalachiteConfig { - MalachiteConfig { - gas_allowance: MalachiteConfig::DEFAULT_GAS_ALLOWANCE, +) -> MalachiteServiceConfig { + MalachiteServiceConfig { + gas_allowance: MalachiteServiceConfig::DEFAULT_GAS_ALLOWANCE, canonical_quarantine: 0, post_quarantine_delay: 0, listen_addr: std::net::SocketAddr::new( diff --git a/ethexe/service/src/config.rs b/ethexe/service/src/config.rs index a0056db7f38..bf89e33a5bf 100644 --- a/ethexe/service/src/config.rs +++ b/ethexe/service/src/config.rs @@ -48,7 +48,7 @@ pub struct MalachiteCliConfig { impl Default for MalachiteCliConfig { fn default() -> Self { Self { - listen_addr: ethexe_malachite::MalachiteConfig::DEFAULT_LISTEN_ADDR, + listen_addr: ethexe_malachite::MalachiteServiceConfig::DEFAULT_LISTEN_ADDR, persistent_peers: Vec::new(), validator_pub_keys: BTreeMap::new(), } diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index d0d3d752982..df3bdab48ab 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -45,7 +45,7 @@ use async_trait::async_trait; use ethexe_blob_loader::{BlobLoader, BlobLoaderEvent, BlobLoaderService, ConsensusLayerConfig}; use ethexe_common::{ CodeAndIdUnchecked, PromiseEmissionMode, - db::{GlobalsStorageRW, MbStorageRO, OnChainStorageRO}, + db::{GlobalsStorageRW, MbStorageRO}, gear::CodeState, injected::{CompactPromise, InjectedTransactionAcceptance, Receipt}, network::VerifiedValidatorMessage, @@ -57,10 +57,12 @@ use ethexe_db::{ }; use ethexe_ethereum::{EthereumBuilder, deploy::EthereumDeployer, router::RouterQuery}; use ethexe_malachite::{ - InjectedTxMempool, MalachiteConfig, MalachiteEvent, MalachiteService, ValidatorEntry, + InjectedTxMempool, MalachiteEvent, MalachiteServiceConfig, MalachiteServiceStarter, + ValidatorEntry, }; use ethexe_network::{ - NetworkEvent, NetworkRuntimeConfig, NetworkService, db_sync::ExternalDataProvider, + NetworkEvent, NetworkRuntimeConfig, NetworkService, TransportType, + db_sync::ExternalDataProvider, }; use ethexe_observer::{ ObserverConfig, ObserverEvent, ObserverService, @@ -165,7 +167,7 @@ pub struct Service { compute: ComputeService, /// `None` for connect (non-validator) nodes. consensus: Option>>, - malachite: Option, + malachite_starter: MalachiteServiceStarter, signer: Signer, // Optional services @@ -358,7 +360,7 @@ impl Service { .await .context("failed to create observer service")?; - let latest_block = observer + let initial_chain_head = observer .block_loader() .load_simple(BlockId::Latest) .await @@ -384,7 +386,7 @@ impl Service { } let validators = router_query - .validators_at(latest_block.hash) + .validators_at(initial_chain_head.hash) .await .context("failed to query validators")?; log::info!("👥 Current validators set: {validators:?}"); @@ -444,24 +446,28 @@ impl Service { }; let network = if let Some(net_config) = &config.network { - // TODO: #4918 create Signer object correctly for test/prod environments - let network_signer = Signer::fs( - config - .node - .key_path - .parent() - .context("key_path has no parent directory")? - .join("net"), - )?; - - let latest_block_data = observer - .block_loader() - .load_simple(BlockId::Latest) - .await - .context("failed to get latest block")?; + let network_signer = match net_config.transport_type { + TransportType::Test => { + let network_signer = Signer::memory(); + let network_private_key = signer + .private_key(net_config.public_key) + .with_context(|| "failed to get test network private key")? + .clone(); + network_signer.import(network_private_key)?; + network_signer + } + TransportType::Default => Signer::fs( + config + .node + .key_path + .parent() + .context("key_path has no parent directory")? + .join("net"), + )?, + }; let runtime_config = NetworkRuntimeConfig { - latest_block_header: latest_block_data.header, + latest_block_header: initial_chain_head.header, latest_validators: validators.clone(), validator_key: validator_pub_key, general_signer: signer.clone(), @@ -490,47 +496,58 @@ impl Service { let compute = ComputeService::with_promise_mode(db.clone(), processor, promises_mode); // Malachite consensus service. - let malachite_home = config - .node - .database_path_for(config.ethereum.router_address) - .join("malachite"); - let mut malachite_base_config = MalachiteConfig::from_home_dir(malachite_home) - .with_listen_addr(config.malachite.listen_addr) - .with_persistent_peers(config.malachite.persistent_peers.clone()); - // Must match the compute layer's quarantine or consensus deadlocks. - malachite_base_config.canonical_quarantine = config.node.canonical_quarantine; - malachite_base_config.post_quarantine_delay = config.node.post_quarantine_delay; - log::info!( - "Malachite listen: {} persistent_peers: {}", - malachite_base_config.listen_addr, - malachite_base_config.persistent_peers.len(), - ); - let malachite = { + + let malachite_starter = { + let malachite_home = config + .node + .database_path_for(config.ethereum.router_address) + .join("malachite"); + let malachite_validator_set = build_malachite_validator_set( validators.iter().copied(), &config.malachite.validator_pub_keys, )?; + + let malachite_service_config = MalachiteServiceConfig::from_home_dir(malachite_home) + .with_listen_addr(config.malachite.listen_addr) + .with_persistent_peers(config.malachite.persistent_peers.clone()) + .with_canonical_quarantine(config.node.canonical_quarantine) + .with_post_quarantine_delay(config.node.post_quarantine_delay) + .with_validators(malachite_validator_set.clone()); + log::info!( - "Malachite validators: {} (local role: {})", - malachite_validator_set.len(), - if validator_pub_key.is_some() { - "validator" - } else { - "full" - }, + "Malachite listen: {} persistent_peers: {}", + malachite_service_config.listen_addr, + malachite_service_config.persistent_peers.len(), ); - let malachite_config = malachite_base_config.with_validators(malachite_validator_set); - Some( - MalachiteService::new( - malachite_config, - db.clone(), - signer.clone(), - validator_pub_key, - std::sync::Arc::new(InjectedTxMempool::new(db.clone())), - ) - .await - .context("failed to start Malachite service")?, + + let validator_config = if let Some(pub_key) = validator_pub_key { + Some(ethexe_malachite::ValidatorConfig { + pub_key, + mempool: InjectedTxMempool::new(db.clone()), + signer: signer.clone(), + }) + } else { + None + }; + + let role = validator_config + .as_ref() + .map(|_| "validator") + .unwrap_or("full"); + log::info!("Malachite not local role: {role}",); + + let malachite_config = + malachite_service_config.with_validators(malachite_validator_set); + + MalachiteServiceStarter::new( + malachite_config, + validator_config, + db.clone(), + initial_chain_head, ) + .await + .context("failed to create Malachite service starter")? }; let fast_sync = config.node.fast_sync; @@ -543,7 +560,7 @@ impl Service { blob_loader, compute, consensus, - malachite, + malachite_starter, signer, prometheus, rpc, @@ -572,7 +589,7 @@ impl Service { compute: ComputeService, signer: Signer, consensus: Option>>, - malachite: Option, + malachite_starter: MalachiteServiceStarter, network: Option, prometheus: Option, rpc: Option, @@ -586,7 +603,7 @@ impl Service { blob_loader, compute, consensus, - malachite, + malachite_starter, signer, network, prometheus, @@ -628,7 +645,7 @@ impl Service { mut blob_loader, mut compute, mut consensus, - mut malachite, + malachite_starter, signer, mut prometheus, rpc, @@ -638,8 +655,11 @@ impl Service { #[cfg(test)] sender, } = self; + let mut shutdown_rx = shutdown_rx; + let malachite = malachite_starter.start().await?; + let (mut rpc_handle, mut rpc) = if let Some(rpc) = rpc { log::info!("🌐 Rpc server starting at: {}", rpc.port()); @@ -667,7 +687,7 @@ impl Service { let event: Event = tokio::select! { event = compute.select_next_some() => event?.into(), event = consensus.maybe_next_some() => event?.into(), - event = malachite.maybe_next_some() => event?.into(), + event = malachite.select_next_some() => event?.into(), event = network.maybe_next_some() => event.into(), event = observer.select_next_some() => event?.into(), event = blob_loader.select_next_some() => event?.into(), @@ -689,38 +709,29 @@ impl Service { match event { Event::Observer(event) => match event { - ObserverEvent::Block(block_data) => { - tracing::info!( - height = %block_data.header.height, - timestamp = %block_data.header.timestamp, - hash = %block_data.hash, - parent_hash = %block_data.header.parent_hash, - "📦 receive a chain head", - ); + ObserverEvent::Block(block) => { + tracing::info!("📦 receive a ethereum chain head: {block}",); + if let Some(c) = consensus.as_mut() { - c.receive_new_chain_head(block_data)?; + c.receive_new_chain_head(block)?; } + + malachite.receive_new_chain_head(block); } - ObserverEvent::BlockSynced(block) => { - log::info!( - "Block synced: {}", - db.block_simple_data(block) - .context("Cannot find header of synced block")? - ); + ObserverEvent::BlockSynced(block_hash) => { + log::info!("Ethereum block synced: {block_hash}"); + + compute.prepare_block(block_hash); - compute.prepare_block(block); if let Some(c) = consensus.as_mut() { - c.receive_synced_block(block)?; + c.receive_synced_block(block_hash)?; } + if let Some(network) = network.as_mut() { - network.set_chain_head(block)?; - } - if let Some(m) = malachite.as_mut() { - let block = db - .block_simple_data(block) - .context("Cannot find header of synced block")?; - m.receive_new_chain_head(block); + network.set_chain_head(block_hash)?; } + + malachite.receive_eb_synced(block_hash); } }, Event::BlobLoader(event) => match event { @@ -736,14 +747,8 @@ impl Service { if let Some(c) = consensus.as_mut() { c.receive_prepared_block(block_hash)?; } - // Malachite's BlockProposal events are gated - // on the EB they advance over being prepared - // (so downstream compute_mb doesn't race the - // code-validation pipeline). Wake the gate - // here so pending events get drained. - if let Some(m) = malachite.as_ref() { - m.receive_eb_prepared(block_hash); - } + + malachite.receive_eb_prepared(block_hash); } ComputeEvent::CodeProcessed(_) => { // Nothing @@ -833,15 +838,14 @@ impl Service { transaction, channel, } => { - let acceptance = match malachite.as_mut() { - Some(malachite) => { - malachite.receive_injected_transaction(*transaction).into() - } - None => InjectedTransactionAcceptance::Reject { - reason: "no malachite service to handle transaction".into(), - }, - }; - let _ = channel.send(acceptance); + let acceptance = + malachite.receive_injected_transaction(*transaction).into(); + if let Err(err) = channel.send(acceptance) { + tracing::error!( + ?err, + "failed to send injected transaction acceptance response" + ) + } } ethexe_network::NetworkInjectedEvent::OutboundAcceptance { transaction_hash, @@ -1055,13 +1059,10 @@ impl Service { } } - // Graceful tear-down: hand the malachite engine a chance to - // flush its WAL and release the RocksDB advisory lock and - // libp2p listener. Without this, an immediate restart on - // the same home directory races the previous lock release. if let Some(m) = malachite.take() { m.shutdown().await; } + Ok(()) } } diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index 267fb33d3de..0a31594312c 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -38,7 +38,7 @@ use ethexe_ethereum::{ router::RouterQuery, }; use ethexe_malachite::{ - InjectedTxMempool, MalachiteConfig, MalachiteService, Multiaddr as MalachiteMultiaddr, PeerId, + InjectedTxMempool, MalachiteServiceConfig, MalachiteService, Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, derive_libp2p_secret, malachite_libp2p_peer_id, }; use ethexe_network::{NetworkConfig, NetworkRuntimeConfig, NetworkService, export::Multiaddr}; @@ -1188,17 +1188,19 @@ impl Node { .path() .to_path_buf(); - let mut mc = MalachiteConfig::from_home_dir(home_path) + let mut mc = MalachiteServiceConfig::from_home_dir(home_path) .with_listen_addr(listen_addr) .with_persistent_peers(persistent_peers) .with_validators(validators); mc.canonical_quarantine = self.canonical_quarantine; mc.post_quarantine_delay = self.post_quarantine_delay; let mempool = std::sync::Arc::new(InjectedTxMempool::new(self.db.clone())); + // Release the port-reservation listener moments before libp2p rebinds. drop(self.malachite_listener.take()); + let svc = MalachiteService::new( - mc, + malach, self.db.clone(), self.signer.clone(), self.validator_config.as_ref().map(|c| c.public_key), From 2ef61645a4c2347414f7363091d859789b1e4e1f Mon Sep 17 00:00:00 2001 From: Grisha Sobol Date: Wed, 10 Jun 2026 21:03:12 +0200 Subject: [PATCH 27/72] fix(vara.eth/malachite): make workspace compile after starter refactor Gate `Signer` re-export behind `std`, await the now-async `try_emit_or_queue`/`receive_*` calls, switch the service event loop and test env to the new `MalachiteServiceStarter` API, drop unused imports. Co-Authored-By: Claude Opus 4.7 --- ethexe/common/src/lib.rs | 4 +- .../malachite/core/tests/multi_validators.rs | 4 +- ethexe/malachite/service/src/externalities.rs | 7 +-- ethexe/malachite/service/src/lib.rs | 2 +- ethexe/malachite/service/src/service.rs | 13 ++---- ethexe/malachite/service/src/starter.rs | 5 +-- ethexe/malachite/service/src/types.rs | 3 -- .../service/tests/restart_resilience.rs | 3 +- ethexe/service/src/lib.rs | 45 +++++++------------ ethexe/service/src/tests/utils/env.rs | 27 ++++++----- 10 files changed, 48 insertions(+), 65 deletions(-) diff --git a/ethexe/common/src/lib.rs b/ethexe/common/src/lib.rs index 35138cfe0cf..a3d55324f4d 100644 --- a/ethexe/common/src/lib.rs +++ b/ethexe/common/src/lib.rs @@ -87,8 +87,10 @@ pub use gsigner::{ }; pub use validators::{EmptyValidatorsError, ValidatorsVec}; pub mod ecdsa { + #[cfg(feature = "std")] + pub use gsigner::secp256k1::Signer; pub use gsigner::secp256k1::{ - ContractSignature, PrivateKey, PublicKey, Signature, SignedData, SignedMessage, Signer, + ContractSignature, PrivateKey, PublicKey, Signature, SignedData, SignedMessage, VerifiedData, }; } diff --git a/ethexe/malachite/core/tests/multi_validators.rs b/ethexe/malachite/core/tests/multi_validators.rs index d6f5c33927d..73bc1f22fce 100644 --- a/ethexe/malachite/core/tests/multi_validators.rs +++ b/ethexe/malachite/core/tests/multi_validators.rs @@ -36,8 +36,8 @@ fn init_tracing() { use anyhow::Result; use async_trait::async_trait; use ethexe_malachite_core::{ - Block, BlockPayload, CommitCertificate, Externalities, H256, MalachiteCoreConfig, MalachiteCore, - Multiaddr, NodeRole, ValidatorEntry, libp2p_peer_id, + Block, BlockPayload, CommitCertificate, Externalities, H256, MalachiteCore, + MalachiteCoreConfig, Multiaddr, NodeRole, ValidatorEntry, libp2p_peer_id, }; use proptest::prelude::*; use tempfile::TempDir; diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index 53c777429e9..904de277e0c 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -53,7 +53,6 @@ use ethexe_common::{ }; use ethexe_db::Database; use ethexe_malachite_core::{Block, BlockPayload, Externalities, MAX_BLOCK_PAYLOAD_BYTES}; -use futures::{FutureExt, pending}; use gprimitives::H256; use parity_scale_codec::{DecodeAll, Encode}; use std::{collections::VecDeque, sync::Arc}; @@ -123,7 +122,8 @@ impl Externalities for EthexeExternalities { mb_hash, }, last_advanced, - ); + ) + .await; Ok(()) } @@ -174,7 +174,8 @@ impl Externalities for EthexeExternalities { mb_hash, }, last_advanced, - ); + ) + .await; Ok(()) } diff --git a/ethexe/malachite/service/src/lib.rs b/ethexe/malachite/service/src/lib.rs index 62fa656671f..7090bbd003f 100644 --- a/ethexe/malachite/service/src/lib.rs +++ b/ethexe/malachite/service/src/lib.rs @@ -61,7 +61,7 @@ use ethexe_common::injected::PurgedTransaction; use gprimitives::H256; pub use crate::{ - config::{MalachiteServiceConfig, ValidatorEntry, ValidatorConfig}, + config::{MalachiteServiceConfig, ValidatorConfig, ValidatorEntry}, mempool::{DEFAULT_POOL_CAPACITY, InjectedTxMempool, Mempool, TxInsertionStatus}, service::MalachiteService, starter::MalachiteServiceStarter, diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 7bb49908de8..5a963178f23 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -26,24 +26,17 @@ use ethexe_common::{ db::{ConfigStorageRO, OnChainStorageRO}, injected::SignedInjectedTransaction, }; -use ethexe_db::Database; use ethexe_malachite_core::MalachiteCore; use futures::{Stream, stream::FusedStream}; use gprimitives::H256; -use gsigner::{ - Signer, - schemes::secp256k1::{PublicKey, Secp256k1}, -}; +use gsigner::schemes::secp256k1::PublicKey; use std::{ collections::HashMap, pin::Pin, - sync::{Arc, RwLock}, + sync::Arc, task::{Context, Poll}, }; -use tokio::sync::{ - Notify, - mpsc::{self, UnboundedReceiver}, -}; +use tokio::sync::mpsc::UnboundedReceiver; /// Public consensus service. pub struct MalachiteService { diff --git a/ethexe/malachite/service/src/starter.rs b/ethexe/malachite/service/src/starter.rs index e5bda9b91d6..635529e2546 100644 --- a/ethexe/malachite/service/src/starter.rs +++ b/ethexe/malachite/service/src/starter.rs @@ -11,10 +11,7 @@ use ethexe_common::{ }; use ethexe_db::Database; use ethexe_malachite_core::{MalachiteCore, MalachiteCoreConfig, NodeRole}; -use gsigner::{ - Signer, - schemes::secp256k1::{PrivateKey, PublicKey, Secp256k1}, -}; +use gsigner::schemes::secp256k1::{PrivateKey, PublicKey}; use std::{collections::HashMap, sync::Arc}; use tokio::sync::{ Notify, RwLock, diff --git a/ethexe/malachite/service/src/types.rs b/ethexe/malachite/service/src/types.rs index 61e25906462..7405a7ada49 100644 --- a/ethexe/malachite/service/src/types.rs +++ b/ethexe/malachite/service/src/types.rs @@ -1,6 +1,4 @@ -use crate::Mempool; use ethexe_common::SimpleBlockData; -use gsigner::schemes::secp256k1::PublicKey; use tokio::sync::{Notify, RwLock}; pub struct ChainHead { @@ -8,4 +6,3 @@ pub struct ChainHead { pub latest_synced: RwLock, pub notify: Notify, } - diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index 7f6fcb7728f..5501857f038 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -28,7 +28,8 @@ use ethexe_common::{ }; use ethexe_db::Database; use ethexe_malachite::{ - MalachiteServiceConfig, MalachiteEvent, MalachiteService, Mempool, TxInsertionStatus, ValidatorEntry, + MalachiteEvent, MalachiteService, MalachiteServiceConfig, Mempool, TxInsertionStatus, + ValidatorEntry, }; use futures::StreamExt as _; use gprimitives::H256; diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index df3bdab48ab..7182216c289 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -521,15 +521,12 @@ impl Service { malachite_service_config.persistent_peers.len(), ); - let validator_config = if let Some(pub_key) = validator_pub_key { - Some(ethexe_malachite::ValidatorConfig { + let validator_config = + validator_pub_key.map(|pub_key| ethexe_malachite::ValidatorConfig { pub_key, mempool: InjectedTxMempool::new(db.clone()), signer: signer.clone(), - }) - } else { - None - }; + }); let role = validator_config .as_ref() @@ -658,7 +655,7 @@ impl Service { let mut shutdown_rx = shutdown_rx; - let malachite = malachite_starter.start().await?; + let mut malachite = malachite_starter.start().await?; let (mut rpc_handle, mut rpc) = if let Some(rpc) = rpc { log::info!("🌐 Rpc server starting at: {}", rpc.port()); @@ -716,7 +713,7 @@ impl Service { c.receive_new_chain_head(block)?; } - malachite.receive_new_chain_head(block); + malachite.receive_new_chain_head(block).await; } ObserverEvent::BlockSynced(block_hash) => { log::info!("Ethereum block synced: {block_hash}"); @@ -731,7 +728,7 @@ impl Service { network.set_chain_head(block_hash)?; } - malachite.receive_eb_synced(block_hash); + malachite.receive_eb_synced(block_hash).await; } }, Event::BlobLoader(event) => match event { @@ -748,7 +745,7 @@ impl Service { c.receive_prepared_block(block_hash)?; } - malachite.receive_eb_prepared(block_hash); + malachite.receive_eb_prepared(block_hash).await; } ComputeEvent::CodeProcessed(_) => { // Nothing @@ -883,18 +880,13 @@ impl Service { transaction, response_sender, } => { - let mut local_acceptance = None; - - if let Some(malachite) = malachite.as_mut() { - let status = - malachite.receive_injected_transaction(transaction.clone()); - local_acceptance = - Some(InjectedTransactionAcceptance::from(status)); - } + let status = + malachite.receive_injected_transaction(transaction.clone()); + let local_acceptance = InjectedTransactionAcceptance::from(status); match network.as_mut() { Some(network) => match local_acceptance { - Some(acceptance @ InjectedTransactionAcceptance::Accept) => { + acceptance @ InjectedTransactionAcceptance::Accept => { // local consensus handle transaction, no need to wait for other acceptances if let Err(err) = network.broadcast_injected_transaction(transaction) @@ -911,7 +903,7 @@ impl Service { } } _ => { - // no local malachite or malachite reject transaction, wait for other acceptances + // local malachite rejected the transaction, wait for other acceptances let tx_hash = transaction.data().to_hash(); if let Some(pending) = network_injected_txs.get_mut(&tx_hash) @@ -925,7 +917,7 @@ impl Service { let pending = PendingNetworkInjectedTx::new( response_sender, pending_responses, - local_acceptance, + Some(local_acceptance), ); network_injected_txs.insert(tx_hash, pending); } @@ -947,12 +939,7 @@ impl Service { }, None => { // No network, send local_acceptance to RPC - let acceptance = local_acceptance.unwrap_or_else(|| { - InjectedTransactionAcceptance::Reject { - reason: "RPC not a validator and do not connect to P2P network".into(), - } - }); - let _ = response_sender.send(acceptance); + let _ = response_sender.send(local_acceptance); } } } @@ -1059,9 +1046,7 @@ impl Service { } } - if let Some(m) = malachite.take() { - m.shutdown().await; - } + malachite.shutdown().await; Ok(()) } diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index 0a31594312c..84e82942026 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -38,8 +38,9 @@ use ethexe_ethereum::{ router::RouterQuery, }; use ethexe_malachite::{ - InjectedTxMempool, MalachiteServiceConfig, MalachiteService, Multiaddr as MalachiteMultiaddr, PeerId, - ValidatorEntry, derive_libp2p_secret, malachite_libp2p_peer_id, + InjectedTxMempool, MalachiteServiceConfig, MalachiteServiceStarter, + Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, derive_libp2p_secret, + malachite_libp2p_peer_id, }; use ethexe_network::{NetworkConfig, NetworkRuntimeConfig, NetworkService, export::Multiaddr}; use ethexe_observer::{ @@ -1194,21 +1195,27 @@ impl Node { .with_validators(validators); mc.canonical_quarantine = self.canonical_quarantine; mc.post_quarantine_delay = self.post_quarantine_delay; - let mempool = std::sync::Arc::new(InjectedTxMempool::new(self.db.clone())); + + let malachite_validator_config = + self.validator_config + .as_ref() + .map(|c| ethexe_malachite::ValidatorConfig { + pub_key: c.public_key, + mempool: InjectedTxMempool::new(self.db.clone()), + signer: self.signer.clone(), + }); // Release the port-reservation listener moments before libp2p rebinds. drop(self.malachite_listener.take()); - let svc = MalachiteService::new( - malach, + MalachiteServiceStarter::new( + mc, + malachite_validator_config, self.db.clone(), - self.signer.clone(), - self.validator_config.as_ref().map(|c| c.public_key), - mempool, + latest_block, ) .await - .expect("MalachiteService::new"); - Some(svc) + .expect("MalachiteServiceStarter::new") }; let (sender, receiver) = events::channel(self.db.clone(), self.kicking_per_blocks.clone()); From 34f3523c23a1fd6dc46084f44b810675a13eca07 Mon Sep 17 00:00:00 2001 From: Grisha Sobol Date: Wed, 10 Jun 2026 21:11:00 +0200 Subject: [PATCH 28/72] test(vara.eth/malachite): re-enable disabled tests, adapt to new internals Drop both `disable-tests` cfg gates; port externalities unit tests to the `ChainHead` register / `Option>` fields and async helpers, port restart_resilience to the `MalachiteServiceStarter` API. Co-Authored-By: Claude Opus 4.7 --- ethexe/malachite/service/src/externalities.rs | 79 +++++++++++-------- .../service/tests/restart_resilience.rs | 48 +++++++---- 2 files changed, 78 insertions(+), 49 deletions(-) diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index 904de277e0c..c9deed84bf4 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -592,17 +592,30 @@ impl EthexeExternalities { } } -#[cfg(feature = "disable-tests")] #[cfg(test)] mod tests { use super::*; use crate::{MalachiteEvent, mempool::EmptyMempool}; use anyhow::Context; use ethexe_common::{ - BlockHeader, + BlockHeader, SimpleBlockData, db::{BlockMetaStorageRW, OnChainStorageRW}, injected::PurgedTransaction, }; + use tokio::sync::{Notify, mpsc}; + + fn make_chain_head() -> Arc { + Arc::new(ChainHead { + latest: RwLock::new(SimpleBlockData::default()), + latest_synced: RwLock::new(SimpleBlockData::default()), + notify: Notify::new(), + }) + } + + async fn set_head(ext: &EthexeExternalities, head: SimpleBlockData) { + *ext.chain_head.latest.write().await = head; + *ext.chain_head.latest_synced.write().await = head; + } fn to_payload(bytes: Vec) -> BlockPayload { BlockPayload::try_from(bytes).expect("test payload within size cap") @@ -640,11 +653,10 @@ mod tests { let (event_tx, event_rx) = mpsc::unbounded_channel(); let ext = EthexeExternalities { db, - mempool: Arc::new(EmptyMempool), - chain_head: Arc::new(RwLock::new(None)), - chain_head_notify: Arc::new(Notify::new()), + mempool: Some(Arc::new(EmptyMempool)), + chain_head: make_chain_head(), event_tx, - pending_events: Mutex::new(VecDeque::new()), + pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 0, @@ -963,7 +975,7 @@ mod tests { header: chain_hashes[2].1, }; let (ext, _rx) = make_externalities(db.clone()); - *ext.chain_head.write().unwrap() = Some(head); + set_head(&ext, head).await; let payload = Operations::new(vec![ Operation::AdvanceTillEthereumBlock { @@ -1052,11 +1064,10 @@ mod tests { let (event_tx, mut event_rx) = mpsc::unbounded_channel(); let ext = EthexeExternalities { db: db.clone(), - mempool: Arc::clone(&tracker) as Arc, - chain_head: Arc::new(RwLock::new(None)), - chain_head_notify: Arc::new(Notify::new()), + mempool: Some(Arc::clone(&tracker) as Arc), + chain_head: make_chain_head(), event_tx, - pending_events: Mutex::new(VecDeque::new()), + pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 0, @@ -1120,11 +1131,10 @@ mod tests { let (event_tx, event_rx) = mpsc::unbounded_channel(); let ext = EthexeExternalities { db, - mempool: mempool as Arc, - chain_head: Arc::new(RwLock::new(None)), - chain_head_notify: Arc::new(Notify::new()), + mempool: Some(mempool as Arc), + chain_head: make_chain_head(), event_tx, - pending_events: Mutex::new(VecDeque::new()), + pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 0, @@ -1266,7 +1276,7 @@ mod tests { assert_eq!(mempool.len(), 1); let (ext, _rx) = make_externalities_with_pool(db, mempool); - *ext.chain_head.write().unwrap() = Some(head); + set_head(&ext, head).await; let payload = ext.build_operations(parent_mb).await.unwrap(); let injected: Vec<_> = payload @@ -1338,7 +1348,7 @@ mod tests { } let (ext, _rx) = make_externalities_with_pool(db.clone(), mempool); - *ext.chain_head.write().unwrap() = Some(head); + set_head(&ext, head).await; // Force AdvanceTillEthereumBlock so eb_touched_programs walks events. // The producer reads chain_head_notify to pick its advance candidate; // since canonical_quarantine = 0, head's parent (block 9) is a valid @@ -1400,7 +1410,7 @@ mod tests { let head = chain.blocks[10].to_simple(); let (ext, _rx) = make_externalities(db.clone()); - *ext.chain_head.write().unwrap() = Some(head); + set_head(&ext, head).await; // Craft an MB payload that adds N/2 fresh destinations on top // of the N/2+1 already touched by EB events → total > limit. @@ -1482,7 +1492,7 @@ mod tests { assert_eq!(mempool.len(), 3); let (ext, _rx) = make_externalities_with_pool(db.clone(), mempool); - *ext.chain_head.write().unwrap() = Some(head); + set_head(&ext, head).await; let payload = ext.build_operations(parent_mb).await.unwrap(); let injected: Vec<_> = payload @@ -1519,7 +1529,7 @@ mod tests { /// Helper: build a tiny chain with one block past quarantine and an /// `EthexeExternalities` whose chain_head points at it. Returns the /// ext + the advance block hash to use for `AdvanceTillEthereumBlock`. - fn chain_with_one_advance( + async fn chain_with_one_advance( db: Database, ) -> ( EthexeExternalities, @@ -1549,7 +1559,7 @@ mod tests { }; let advance_hash = chain_hashes[1].0; let (ext, rx) = make_externalities(db); - *ext.chain_head.write().unwrap() = Some(head); + set_head(&ext, head).await; (ext, rx, advance_hash) } @@ -1561,7 +1571,7 @@ mod tests { #[tokio::test] async fn validate_rejects_gas_allowance_above_default() { let db = Database::memory(); - let (ext, _rx, advance) = chain_with_one_advance(db); + let (ext, _rx, advance) = chain_with_one_advance(db).await; let payload = Operations::new(vec![ Operation::AdvanceTillEthereumBlock { block_hash: advance, @@ -1585,7 +1595,7 @@ mod tests { #[tokio::test] async fn validate_rejects_mb_missing_progress_tasks() { let db = Database::memory(); - let (ext, _rx, advance) = chain_with_one_advance(db); + let (ext, _rx, advance) = chain_with_one_advance(db).await; let payload = Operations::new(vec![ Operation::AdvanceTillEthereumBlock { block_hash: advance, @@ -1607,7 +1617,7 @@ mod tests { #[tokio::test] async fn validate_rejects_mb_missing_process_queues() { let db = Database::memory(); - let (ext, _rx, advance) = chain_with_one_advance(db); + let (ext, _rx, advance) = chain_with_one_advance(db).await; let payload = Operations::new(vec![ Operation::AdvanceTillEthereumBlock { block_hash: advance, @@ -1642,7 +1652,7 @@ mod tests { db.globals_mutate(|g| g.latest_computed_mb_hash = parent_mb); let (ext, _rx) = make_externalities(db.clone()); - *ext.chain_head.write().unwrap() = Some(head); + set_head(&ext, head).await; let pk = PrivateKey::random(); let tx = SignedMessage::create( @@ -1677,7 +1687,7 @@ mod tests { #[tokio::test] async fn validate_rejects_process_queues_not_last() { let db = Database::memory(); - let (ext, _rx, advance) = chain_with_one_advance(db); + let (ext, _rx, advance) = chain_with_one_advance(db).await; let payload = Operations::new(vec![ Operation::AdvanceTillEthereumBlock { block_hash: advance, @@ -1744,7 +1754,7 @@ mod tests { }); let (ext, _rx) = make_externalities(db.clone()); - *ext.chain_head.write().unwrap() = Some(head); + set_head(&ext, head).await; // MB proposes Advance to chain[1] — strictly older than chain[3] // (parent's last_advanced_eb). `verify_passed` accepts (chain[1] @@ -1807,7 +1817,7 @@ mod tests { let stranger_advance = H256::from([0xEE; 32]); let (ext, _rx) = make_externalities(db.clone()); - *ext.chain_head.write().unwrap() = Some(head); + set_head(&ext, head).await; let payload = Operations::new(vec![ Operation::AdvanceTillEthereumBlock { @@ -1834,8 +1844,8 @@ mod tests { /// from the local chain head. With `(canonical_quarantine = 2, /// post_quarantine_delay = 3)` the candidate sits 5 blocks below /// head — exactly what we verify by walking parent links. - #[test] - fn producer_picks_anchor_at_canonical_plus_post_delay() { + #[tokio::test] + async fn producer_picks_anchor_at_canonical_plus_post_delay() { use ethexe_common::db::OnChainStorageRO; let db = Database::memory(); @@ -1873,20 +1883,21 @@ mod tests { let (event_tx, _event_rx) = mpsc::unbounded_channel(); let ext = EthexeExternalities { db: db.clone(), - mempool: Arc::new(EmptyMempool), - chain_head: Arc::new(RwLock::new(Some(head))), - chain_head_notify: Arc::new(Notify::new()), + mempool: Some(Arc::new(EmptyMempool)), + chain_head: make_chain_head(), event_tx, - pending_events: Mutex::new(VecDeque::new()), + pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 2, post_quarantine_delay: 3, }, }; + set_head(&ext, head).await; let candidate = ext .find_eb_candidate_for_advancing(H256::zero()) + .await .expect("must surface a candidate — chain is deep enough"); // Walk back `2 + 3 = 5` parents from head; that's the expected // anchor. diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index 5501857f038..9d925733703 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -16,9 +16,7 @@ //! `globals.latest_finalized_mb_hash` is gap-free across the //! restart boundary, and the latest pointer never rewinds. -#![cfg(feature = "disable-tests")] - -use std::{path::Path, sync::Arc, time::Duration}; +use std::{path::Path, time::Duration}; use async_trait::async_trait; use ethexe_common::{ @@ -28,8 +26,8 @@ use ethexe_common::{ }; use ethexe_db::Database; use ethexe_malachite::{ - MalachiteEvent, MalachiteService, MalachiteServiceConfig, Mempool, TxInsertionStatus, - ValidatorEntry, + MalachiteEvent, MalachiteService, MalachiteServiceConfig, MalachiteServiceStarter, Mempool, + TxInsertionStatus, ValidatorConfig, ValidatorEntry, }; use futures::StreamExt as _; use gprimitives::H256; @@ -143,6 +141,14 @@ fn build_config( } } +/// Feed one chain head into the service: register it as the new head +/// and immediately mark it synced (the test seeds all headers/events +/// upfront, so "observed" and "synced" coincide here). +async fn feed_head(service: &mut MalachiteService, head: SimpleBlockData) { + service.receive_new_chain_head(head).await; + service.receive_eb_synced(head.hash).await; +} + /// Drain the service stream until at least `target` finalize events /// have been observed or `budget` elapses. Each round of the loop /// feeds the next chain head from `pending_heads` BEFORE polling, so @@ -165,7 +171,7 @@ async fn collect_until_finalized( // Push the first head right away so the producer can build the // genesis MB. if let Some(head) = pending_heads.next() { - service.receive_new_chain_head(head); + feed_head(service, head).await; } while tokio::time::Instant::now() < deadline { let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); @@ -182,7 +188,7 @@ async fn collect_until_finalized( // next round, so its quarantine-advance candidate // moves forward. if let Some(head) = pending_heads.next() { - service.receive_new_chain_head(head); + feed_head(service, head).await; } } Ok(Some(Ok(MalachiteEvent::BlockProposal { .. }))) => { @@ -216,14 +222,20 @@ async fn single_validator_finalizes_and_recovers_after_restart() { let (signer, pub_key) = build_signer(home.path()); // ---- first run ------------------------------------------------- - let mut svc = MalachiteService::new( + let mut svc = MalachiteServiceStarter::new( build_config(home.path(), 30_001, pub_key), + Some(ValidatorConfig { + pub_key, + mempool: EmptyMempool, + signer: signer.clone(), + }), db.clone(), - signer.clone(), - Some(pub_key), - Arc::new(EmptyMempool), + chain[0], ) .await + .expect("create malachite service starter") + .start() + .await .expect("start malachite service"); // Feed chain heads one-per-round so the quarantine-advance @@ -258,14 +270,20 @@ async fn single_validator_finalizes_and_recovers_after_restart() { tokio::time::sleep(Duration::from_millis(500)).await; // ---- second run on the SAME home dir + DB ---------------------- - let mut svc2 = MalachiteService::new( + let mut svc2 = MalachiteServiceStarter::new( build_config(home.path(), 30_001, pub_key), + Some(ValidatorConfig { + pub_key, + mempool: EmptyMempool, + signer, + }), db.clone(), - signer, - Some(pub_key), - Arc::new(EmptyMempool), + chain[31], ) .await + .expect("create malachite service starter after restart") + .start() + .await .expect("restart malachite service"); let mut pending2 = chain[32..].iter().copied(); let (high2, finalized2) = From 90580d1444968998badf0ca79a3a71adb6d78b83 Mon Sep 17 00:00:00 2001 From: Grisha Sobol Date: Wed, 10 Jun 2026 21:22:41 +0200 Subject: [PATCH 29/72] fix(vara.eth/malachite): publish synced head to externalities on eb-synced `ChainHead::latest_synced` was only seeded once in the starter, so the producer's quarantine probe never saw new EBs and proposals timed out. Update it in `receive_eb_synced` before waking the producer. Co-Authored-By: Claude Opus 4.7 --- ethexe/malachite/service/src/service.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 5a963178f23..dd616b2be51 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -83,6 +83,11 @@ impl MalachiteService { return; } + // Publish the synced head for the externalities' quarantine + // checks BEFORE waking the producer, so the woken round + // already sees it. + *self.chain_head.latest_synced.write().await = chain_head; + // Notify inner proposer if it waits (see EthexeExternalities::wait_for_proposable_content) self.chain_head.notify.notify_one(); From 1a3ab40ee0e79071643334acbfafaa4a22791641 Mon Sep 17 00:00:00 2001 From: Gregory Sobol Date: Wed, 10 Jun 2026 21:37:58 +0200 Subject: [PATCH 30/72] malachite core acceptance fixing --- Cargo.lock | 1 + ethexe/common/src/primitives.rs | 6 ++ ethexe/malachite/core/Cargo.toml | 21 ++--- ethexe/malachite/core/src/app.rs | 100 +++++++++++++-------- ethexe/malachite/core/src/externalities.rs | 8 +- 5 files changed, 78 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef2fac30c1a..abc97d909a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5712,6 +5712,7 @@ dependencies = [ "arc-malachitebft-test", "async-trait", "bytes", + "ethexe-common", "futures", "gear-core", "gear-workspace-hack", diff --git a/ethexe/common/src/primitives.rs b/ethexe/common/src/primitives.rs index 13cc16ef0ea..cb8a99cf2f8 100644 --- a/ethexe/common/src/primitives.rs +++ b/ethexe/common/src/primitives.rs @@ -233,6 +233,12 @@ pub type ScheduledTask = gear_core::tasks::ScheduledTask; /// Scheduler; (block height, scheduled task) pub type Schedule = BTreeMap>; +#[derive(Debug, Clone, PartialEq, Eq, derive_more::Display, derive_more::IsVariant, Encode, Decode, TypeInfo)] +pub enum Acceptance { + Accepted(A), + Rejected(R), +} + #[cfg(test)] mod tests { use super::*; diff --git a/ethexe/malachite/core/Cargo.toml b/ethexe/malachite/core/Cargo.toml index 0a9f1290f84..3d419bd8b64 100644 --- a/ethexe/malachite/core/Cargo.toml +++ b/ethexe/malachite/core/Cargo.toml @@ -9,7 +9,12 @@ homepage.workspace = true repository.workspace = true [dependencies] -# Malachite BFT engine +gear-core = { workspace = true, features = ["std"] } +gprimitives = { workspace = true, features = ["std"] } +gsigner = { workspace = true, features = ["std", "secp256k1", "codec", "serde", "keyring"] } +ethexe-common = { workspace = true, features = ["std"] } +gear-workspace-hack.workspace = true + malachitebft-app-channel.workspace = true malachitebft-app.workspace = true malachitebft-codec.workspace = true @@ -21,7 +26,6 @@ malachitebft-signing-ecdsa.workspace = true malachitebft-sync.workspace = true malachitebft-test.workspace = true -# Async + utilities advisory-lock.workspace = true anyhow.workspace = true async-trait.workspace = true @@ -33,22 +37,9 @@ serde = { workspace = true, features = ["derive"] } sha3 = { workspace = true, features = ["std"] } tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } tracing.workspace = true - -# Crypto + libp2p (kept ethexe-shaped per design: secp256k1 + 20-byte addresses). -# Address type is reused from gsigner so the application side (ethexe today, -# anything else tomorrow) gets the same address shape it already deals with. -gear-core = { workspace = true, features = ["std"] } -gprimitives = { workspace = true, features = ["std"] } -gsigner = { workspace = true, features = ["std", "secp256k1", "codec", "serde", "keyring"] } libp2p-identity.workspace = true - -# Persistent internal store. Version pinned to match ethexe-db's -# librocksdb-sys (only one `links = "rocksdb"` crate is allowed in -# the dependency graph). rocksdb.workspace = true -gear-workspace-hack.workspace = true - [dev-dependencies] advisory-lock.workspace = true proptest.workspace = true diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index 9f2452741ae..78b9178714f 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -37,8 +37,9 @@ use crate::{ streaming::ProposalParts, types::{Address, Block, CommitCertificate, H256}, }; -use anyhow::{Context as _, Result, anyhow}; +use anyhow::{Context as _, Result, anyhow, ensure}; use bytes::Bytes; +use ethexe_common::Acceptance; use malachitebft_app_channel::{ AppMsg, Channels, NetworkMsg, app::{ @@ -55,7 +56,7 @@ use malachitebft_app_channel::{ use malachitebft_core_types::Height as _; use parity_scale_codec::{Decode, Encode}; use std::{ops::RangeInclusive, sync::Arc}; -use tracing::{error, info}; +use tracing::{debug, error, info, trace, warn}; /// Max allowed distance into the future for pending proposal parts. const FUTURE_HEIGHT_WINDOW: u64 = 4; @@ -188,12 +189,12 @@ where StreamContent::Data(p) => p.get_type(), StreamContent::Fin => "fin", }; - info!(%from, %part.sequence, part.type = %part_type, "ReceivedProposalPart"); + trace!(%from, %part.sequence, part.type = %part_type, "ReceivedProposalPart"); let value = self .process_received_proposal_part(from, part) .await .unwrap_or_else(|e| { - error!(?e, "ReceivedProposalPart: process failed"); + error!("ReceivedProposalPart: process failed, {e}"); None }); if reply.send(value).is_err() { @@ -346,12 +347,17 @@ where let pending = self.state.store.get_pending_proposal_parts(height, round)?; for parts in pending { let promote = async { - let proposed = self.assemble_and_validate(&parts).await?; + let (proposed, block) = match self.assemble_and_validate(&parts).await? { + Acceptance::Accepted(res) => res, + Acceptance::Rejected(reason) => { + warn!(%height, %round, reason, "rejecting pending proposal"); + return Ok(()); + } + }; self.state.store.store_undecided_proposal(&proposed)?; - let block = Block::decode(&mut &proposed.value.block_bytes[..]) - .map_err(|e| anyhow!("decoding Block from pending proposal: {e}"))?; self.record_assembled_block(block).await }; + if let Err(e) = promote.await { error!(?e, "rejecting invalid pending proposal"); } @@ -445,10 +451,14 @@ where .store_pending_proposal_parts(&parts, &value_id)?; Ok(None) } else { - let proposed = self.assemble_and_validate(&parts).await?; + let (proposed, block) = match self.assemble_and_validate(&parts).await? { + Acceptance::Accepted(res) => res, + Acceptance::Rejected(reason) => { + warn!(%parts.height, %parts.round, reason, "rejecting received proposal"); + return Ok(None); + } + }; self.state.store.store_undecided_proposal(&proposed)?; - let block = Block::decode(&mut &proposed.value.block_bytes[..]) - .map_err(|e| anyhow!("decoding Block from received proposal: {e}"))?; self.record_assembled_block(block).await?; Ok(Some(proposed)) } @@ -571,17 +581,25 @@ where async fn assemble_and_validate( &self, parts: &ProposalParts, - ) -> Result> { + ) -> Result, Block), String>> { let proposed = State::assemble_value_from_parts(parts.clone())?; - let block = Block::decode(&mut &proposed.value.block_bytes[..]) - .map_err(|e| anyhow!("decoding Block from value bytes: {e}"))?; + + let block = match Block::decode(&mut &proposed.value.block_bytes[..]) { + Ok(b) => b, + Err(e) => { + let reason = format!("unable to decode Block from proposal: {e}"); + return Ok(Acceptance::Rejected(reason)); + } + }; + if block.height != proposed.height.as_u64() { - return Err(anyhow!( + let reason = format!( "block.height ({}) does not match proposed height ({})", - block.height, - proposed.height - )); + block.height, proposed.height + ); + return Ok(Acceptance::Rejected(reason)); } + let local_parent = if proposed.height.as_u64() <= 1 { H256::zero() } else { @@ -595,29 +613,25 @@ where ) })? }; + if block.parent_hash != local_parent { - return Err(anyhow!( - "parent_hash mismatch at height {}: block claims {:?}, local view {:?}", - proposed.height, - block.parent_hash, - local_parent - )); + let reason = format!( + "proposal parent_hash mismatch at height {}: block claims {:?}, local view {:?}", + proposed.height, block.parent_hash, local_parent + ); + return Ok(Acceptance::Rejected(reason)); } - // Parent + height already validated above. The application - // only sees the parent hash + payload — payload-level checks - // live there. - let valid = self + + let validation_status = self .externalities - .validate_block_above(block.parent_hash, block.payload) + .validate_block_above(block.parent_hash, &block.payload) .await - .context("Externalities::validate_block_above")?; - if !valid { - return Err(anyhow!( - "application rejected proposal at height {}", - proposed.height - )); + .context("assemble_and_validate: validate block")?; + + match validation_status { + Acceptance::Accepted(()) => Ok(Acceptance::Accepted((proposed, block))), + Acceptance::Rejected(reason) => Ok(Acceptance::Rejected(reason)), } - Ok(proposed) } /// Drive the strict-ordering save cascade against the application @@ -794,8 +808,12 @@ mod tests { async fn build_block_above(&self, _: H256) -> Result { Ok(test_payload()) } - async fn validate_block_above(&self, _: H256, _: BlockPayload) -> Result { - Ok(true) + async fn validate_block_above( + &self, + _: H256, + _: &BlockPayload, + ) -> Result> { + Ok(Acceptance::Accepted(())) } } @@ -977,8 +995,12 @@ mod tests { async fn build_block_above(&self, _: H256) -> Result { Ok(test_payload()) } - async fn validate_block_above(&self, _: H256, _: BlockPayload) -> Result { - Ok(true) + async fn validate_block_above( + &self, + _: H256, + _: &BlockPayload, + ) -> Result> { + Ok(Acceptance::Accepted(())) } } diff --git a/ethexe/malachite/core/src/externalities.rs b/ethexe/malachite/core/src/externalities.rs index 87a494e317f..05b8f0c448d 100644 --- a/ethexe/malachite/core/src/externalities.rs +++ b/ethexe/malachite/core/src/externalities.rs @@ -3,10 +3,10 @@ //! Application callbacks the service makes to the outside world. +use crate::types::{Block, BlockPayload, CommitCertificate, H256}; use anyhow::Result; use async_trait::async_trait; - -use crate::types::{Block, BlockPayload, CommitCertificate, H256}; +use ethexe_common::Acceptance; /// Application-side callbacks the consensus service requires. /// @@ -99,6 +99,6 @@ pub trait Externalities: Send + Sync + 'static { async fn validate_block_above( &self, parent_mb_hash: H256, - payload: BlockPayload, - ) -> Result; + payload: &BlockPayload, + ) -> Result>; } From 09b036f9afbc6dbc573abf85731306b6fe69bf7d Mon Sep 17 00:00:00 2001 From: Grisha Sobol Date: Wed, 10 Jun 2026 21:53:48 +0200 Subject: [PATCH 31/72] refactor(vara.eth/malachite): adapt service and tests to Acceptance-based validation `Externalities::validate_block_above` now takes `&BlockPayload` and returns `Acceptance<(), String>`; rejection reasons travel in the variant (logged by the core) instead of local `warn!` + `Ok(false)`. Port the multi_validators TestExt impl and drop imports left unused by the core change. Co-Authored-By: Claude Opus 4.7 --- ethexe/common/src/primitives.rs | 12 ++- ethexe/malachite/core/src/app.rs | 4 +- .../malachite/core/tests/multi_validators.rs | 7 +- ethexe/malachite/service/src/externalities.rs | 99 +++++++++---------- 4 files changed, 64 insertions(+), 58 deletions(-) diff --git a/ethexe/common/src/primitives.rs b/ethexe/common/src/primitives.rs index cb8a99cf2f8..e2bd0ad3eec 100644 --- a/ethexe/common/src/primitives.rs +++ b/ethexe/common/src/primitives.rs @@ -233,7 +233,17 @@ pub type ScheduledTask = gear_core::tasks::ScheduledTask; /// Scheduler; (block height, scheduled task) pub type Schedule = BTreeMap>; -#[derive(Debug, Clone, PartialEq, Eq, derive_more::Display, derive_more::IsVariant, Encode, Decode, TypeInfo)] +#[derive( + Debug, + Clone, + PartialEq, + Eq, + derive_more::Display, + derive_more::IsVariant, + Encode, + Decode, + TypeInfo, +)] pub enum Acceptance { Accepted(A), Rejected(R), diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index 78b9178714f..47b1f73de0e 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -37,7 +37,7 @@ use crate::{ streaming::ProposalParts, types::{Address, Block, CommitCertificate, H256}, }; -use anyhow::{Context as _, Result, anyhow, ensure}; +use anyhow::{Context as _, Result, anyhow}; use bytes::Bytes; use ethexe_common::Acceptance; use malachitebft_app_channel::{ @@ -56,7 +56,7 @@ use malachitebft_app_channel::{ use malachitebft_core_types::Height as _; use parity_scale_codec::{Decode, Encode}; use std::{ops::RangeInclusive, sync::Arc}; -use tracing::{debug, error, info, trace, warn}; +use tracing::{error, info, trace, warn}; /// Max allowed distance into the future for pending proposal parts. const FUTURE_HEIGHT_WINDOW: u64 = 4; diff --git a/ethexe/malachite/core/tests/multi_validators.rs b/ethexe/malachite/core/tests/multi_validators.rs index 73bc1f22fce..b7d7569059b 100644 --- a/ethexe/malachite/core/tests/multi_validators.rs +++ b/ethexe/malachite/core/tests/multi_validators.rs @@ -35,6 +35,7 @@ fn init_tracing() { use anyhow::Result; use async_trait::async_trait; +use ethexe_common::Acceptance; use ethexe_malachite_core::{ Block, BlockPayload, CommitCertificate, Externalities, H256, MalachiteCore, MalachiteCoreConfig, Multiaddr, NodeRole, ValidatorEntry, libp2p_peer_id, @@ -177,8 +178,8 @@ impl Externalities for TestExt { async fn validate_block_above( &self, parent_hash: H256, - _payload: BlockPayload, - ) -> Result { + _payload: &BlockPayload, + ) -> Result> { let mut s = self.state.lock().unwrap(); if let Some(last_fin) = s.finalized.last().copied() && parent_hash != last_fin @@ -187,7 +188,7 @@ impl Externalities for TestExt { "validate_block_above: parent_hash mismatch — expected {last_fin:?}, got {parent_hash:?}" )); } - Ok(true) + Ok(Acceptance::Accepted(())) } } diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index c9deed84bf4..aef06f56c4d 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -46,7 +46,7 @@ use crate::{ use anyhow::{Result, anyhow, ensure}; use async_trait::async_trait; use ethexe_common::{ - MAX_TOUCHED_PROGRAMS_PER_MB, + Acceptance, MAX_TOUCHED_PROGRAMS_PER_MB, db::{CompactMb, GlobalsStorageRO, GlobalsStorageRW, MbStorageRO, MbStorageRW}, injected::{MAX_INJECTED_TRANSACTIONS_SIZE_PER_MB, SignedInjectedTransaction}, malachite::{Operation, Operations}, @@ -287,12 +287,17 @@ impl Externalities for EthexeExternalities { }) } - async fn validate_block_above(&self, parent_hash: H256, payload: BlockPayload) -> Result { + async fn validate_block_above( + &self, + parent_hash: H256, + payload: &BlockPayload, + ) -> Result> { let payload = match Operations::decode_all(&mut payload.as_ref()) { Ok(payload) => payload, Err(e) => { - warn!(error = %e, "validate: undecodable block payload — rejecting"); - return Ok(false); + return Ok(Acceptance::Rejected(format!( + "undecodable block payload: {e}" + ))); } }; @@ -314,30 +319,29 @@ impl Externalities for EthexeExternalities { } let Some(Operation::ProgressTasks) = next else { - warn!( - "validate: MB shape violation — expected `ProgressTasks` bookend, got {:?}", + return Ok(Acceptance::Rejected(format!( + "MB shape violation — expected `ProgressTasks` bookend, got {:?}", next.map(|t| t.tag()) - ); - return Ok(false); + ))); }; let Some(Operation::ProcessQueues { gas_allowance }) = iter.next() else { - warn!("validate: MB shape violation — expected `ProcessQueues` bookend"); - return Ok(false); + return Ok(Acceptance::Rejected( + "MB shape violation — expected `ProcessQueues` bookend".to_string(), + )); }; if *gas_allowance > crate::MalachiteServiceConfig::DEFAULT_GAS_ALLOWANCE { - warn!( - allowance = *gas_allowance, - cap = crate::MalachiteServiceConfig::DEFAULT_GAS_ALLOWANCE, - "validate: ProcessQueues.gas_allowance exceeds protocol cap" - ); - return Ok(false); + return Ok(Acceptance::Rejected(format!( + "ProcessQueues.gas_allowance {gas_allowance} exceeds protocol cap {}", + crate::MalachiteServiceConfig::DEFAULT_GAS_ALLOWANCE + ))); } if iter.next().is_some() { - warn!("validate: MB has extra operations after the `ProcessQueues` bookend"); - return Ok(false); + return Ok(Acceptance::Rejected( + "MB has extra operations after the `ProcessQueues` bookend".to_string(), + )); } // TODO: #5477 extract a shared `check_eb_advance` helper so this @@ -364,13 +368,10 @@ impl Externalities for EthexeExternalities { self.cfg.canonical_quarantine, start_block_hash, ) { - warn!( - error = %e, - %advance, - parent_advanced = %parent_advanced, - "validate: advance not yet covered by local view — rejecting", - ); - return Ok(false); + return Ok(Acceptance::Rejected(format!( + "advance {advance} (parent_advanced {parent_advanced}) \ + not yet covered by local view: {e}" + ))); } match quarantine::is_strict_descendant_of( @@ -381,21 +382,16 @@ impl Externalities for EthexeExternalities { ) { Ok(true) => {} Ok(false) => { - warn!( - %advance, - parent_advanced = %parent_advanced, - "validate: advance not strict descendant of parent.last_advanced_eb — rejecting", - ); - return Ok(false); + return Ok(Acceptance::Rejected(format!( + "advance {advance} is not a strict descendant of \ + parent.last_advanced_eb {parent_advanced}" + ))); } Err(e) => { - warn!( - error = %e, - %advance, - parent_advanced = %parent_advanced, - "validate: is_strict_descendant_of failed — rejecting", - ); - return Ok(false); + return Ok(Acceptance::Rejected(format!( + "is_strict_descendant_of failed for advance {advance} \ + (parent_advanced {parent_advanced}): {e}" + ))); } } } @@ -410,12 +406,10 @@ impl Externalities for EthexeExternalities { match checker.check_tx_validity(signed)? { TxValidity::Valid => {} reason => { - warn!( - tx_hash = %signed.data().to_hash(), - ?reason, - "validate: injected tx fails TxValidity — rejecting MB", - ); - return Ok(false); + return Ok(Acceptance::Rejected(format!( + "injected tx {} fails TxValidity: {reason:?}", + signed.data().to_hash() + ))); } } } @@ -459,14 +453,13 @@ impl Externalities for EthexeExternalities { } } if touched.len() > limit { - warn!( - touched = touched.len(), - limit, "validate: MB touches too many programs — rejecting" - ); - return Ok(false); + return Ok(Acceptance::Rejected(format!( + "MB touches too many programs: {} > limit {limit}", + touched.len() + ))); } - Ok(true) + Ok(Acceptance::Accepted(())) } } @@ -627,8 +620,9 @@ mod tests { /// Mirrors the producer-side encoding step the inner core service /// applies to whatever `build_block_above` returns. async fn validate_operations(&self, parent: H256, ops: Operations) -> Result { - self.validate_block_above(parent, to_payload(ops.encode())) + self.validate_block_above(parent, &to_payload(ops.encode())) .await + .map(|acceptance| acceptance.is_accepted()) } /// Test-only inverse of [`Self::validate_operations`]: run the @@ -901,9 +895,10 @@ mod tests { let mut bytes = payload(None, 1).encode(); bytes.extend_from_slice(&[0u8; 16]); assert!( - !ext.validate_block_above(H256::zero(), to_payload(bytes)) + ext.validate_block_above(H256::zero(), &to_payload(bytes)) .await .unwrap() + .is_rejected() ); } From 6543f0abb927819001e126d5947c677ced483050 Mon Sep 17 00:00:00 2001 From: Gregory Sobol Date: Thu, 11 Jun 2026 13:36:33 +0200 Subject: [PATCH 32/72] fixing errors handling --- ethexe/malachite/service/src/externalities.rs | 174 +++++++++--------- ethexe/malachite/service/src/mempool.rs | 65 +++---- ethexe/malachite/service/src/quarantine.rs | 168 ++++++----------- ethexe/malachite/service/src/service.rs | 43 +++-- ethexe/malachite/service/src/types.rs | 3 +- .../service/tests/restart_resilience.rs | 4 +- ethexe/service/src/lib.rs | 20 +- 7 files changed, 217 insertions(+), 260 deletions(-) diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index aef06f56c4d..748bf8ca312 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -43,11 +43,13 @@ use crate::{ tx_validity::{TxValidity, TxValidityChecker, eb_touched_programs}, types::ChainHead, }; -use anyhow::{Result, anyhow, ensure}; +use anyhow::{Context, Result, anyhow, ensure}; use async_trait::async_trait; use ethexe_common::{ Acceptance, MAX_TOUCHED_PROGRAMS_PER_MB, - db::{CompactMb, GlobalsStorageRO, GlobalsStorageRW, MbStorageRO, MbStorageRW}, + db::{ + CompactMb, GlobalsStorageRO, GlobalsStorageRW, MbStorageRO, MbStorageRW, OnChainStorageRO, + }, injected::{MAX_INJECTED_TRANSACTIONS_SIZE_PER_MB, SignedInjectedTransaction}, malachite::{Operation, Operations}, }; @@ -57,7 +59,7 @@ use gprimitives::H256; use parity_scale_codec::{DecodeAll, Encode}; use std::{collections::VecDeque, sync::Arc}; use tokio::sync::{RwLock, mpsc::UnboundedSender}; -use tracing::{error, info, warn}; +use tracing::{debug, error, trace, warn}; pub struct ExternalitiesConfig { pub gas_allowance: u64, @@ -89,11 +91,10 @@ impl Externalities for EthexeExternalities { let parent = mb.parent_hash; - let parent_advanced = if parent.is_zero() { - H256::zero() - } else { - self.db.mb_meta(parent).last_advanced_eb - }; + let parent_advanced = parent + .is_zero() + .then(H256::zero) + .unwrap_or_else(|| self.db.mb_meta(parent).last_advanced_eb); let last_advanced = payload .iter() .rev() @@ -132,27 +133,27 @@ impl Externalities for EthexeExternalities { mb_hash: H256, cert: ethexe_malachite_core::CommitCertificate, ) -> Result<()> { - let compact = self.db.mb_compact_block(mb_hash).ok_or_else(|| { - anyhow!( - "process_mb_finalized: no CompactMb for {mb_hash} \ - (process_mb_proposal must run first)" - ) - })?; - let payload = self.db.operations(compact.operations_hash).ok_or_else(|| { - anyhow!( - "mark_finalized: operations blob {} missing for block {mb_hash}", - compact.operations_hash - ) - })?; - if let Some(pool) = self.mempool.as_ref() { - let injected: Vec = payload - .iter() - .filter_map(|tx| match tx { - Operation::Injected(t) => Some(t.clone()), + // Remove any finalized MB's proposed injected txs from the mempool. + + let compact = self + .db + .mb_compact_block(mb_hash) + .with_context(|| format!("no CompactMb for {mb_hash}"))?; + + let operations = self + .db + .operations(compact.operations_hash) + .with_context(|| format!("operations blob missing for block {mb_hash}"))?; + + let injected: Vec = operations + .into_iter() + .filter_map(|op| match op { + Operation::Injected(tx) => Some(tx), _ => None, }) .collect(); + if !injected.is_empty() { pool.forget(&injected).await; } @@ -176,24 +177,23 @@ impl Externalities for EthexeExternalities { last_advanced, ) .await; + Ok(()) } async fn build_block_above(&self, parent_mb_hash: H256) -> Result { ensure!( self.mempool.is_some(), - "build_block_above cannot be called when node is not validator" + "build_block_above must not be called when node is not validator" ); - let parent_advanced = if parent_mb_hash.is_zero() { - H256::zero() - } else { - self.db.mb_meta(parent_mb_hash).last_advanced_eb - }; - - let (advance, injected) = self.wait_for_proposable_content(parent_advanced).await; + let parent_advanced = parent_mb_hash + .is_zero() + .then(H256::zero) + .unwrap_or_else(|| self.db.mb_meta(parent_mb_hash).last_advanced_eb); + let (advance, injected) = self.wait_for_proposable_content(parent_advanced).await?; - info!( + debug!( %parent_mb_hash, %parent_advanced, advance = ?advance, @@ -201,6 +201,7 @@ impl Externalities for EthexeExternalities { "build_block_above: proposable content resolved", ); + // Filter the fetched injected txs down to the valid ones before we start MB assembly let valid_injected_txs = { let chain_head = *self.chain_head.latest_synced.read().await; let checker = @@ -210,7 +211,7 @@ impl Externalities for EthexeExternalities { match checker.check_tx_validity(&tx)? { TxValidity::Valid => accepted.push(tx), reason => { - warn!( + debug!( tx_hash = %tx.data().to_hash(), ?reason, "build_block_above: dropping injected tx — fails TxValidity", @@ -230,7 +231,7 @@ impl Externalities for EthexeExternalities { // Producer can't shrink this — the EB events themselves // already exceed the cap. Drop injected txs and let the // MB advance the EB anyway so the chain progresses. - warn!( + error!( initial_touched_count, limit = MAX_TOUCHED_PROGRAMS_PER_MB, "build_block_above: EB events already exceed touched-programs cap; \ @@ -238,6 +239,7 @@ impl Externalities for EthexeExternalities { ); } + // Cap the injected txs to stay within the remaining limits let mut size_counter: usize = 0; let mut capped_injected_txs: Vec = Vec::with_capacity(valid_injected_txs.len()); @@ -351,53 +353,51 @@ impl Externalities for EthexeExternalities { // each early-return below so operators can tune // `post_quarantine_delay` from observability rather than logs. + // Take latest synced EB as the reference point + // for all the quarantine and transactions checks below + let chain_head = *self.chain_head.latest_synced.read().await; + // Advanced block quarantine checks if let Some(advance) = advance { - let parent_advanced = if parent_hash.is_zero() { - H256::zero() - } else { - self.db.mb_meta(parent_hash).last_advanced_eb + let Some(advance) = self.db.block_simple_data(advance) else { + return Ok(Acceptance::Rejected(format!( + "advance EB {advance} not found in local DB" + ))); }; - let start_block_hash = self.db.globals().start_block_hash; - let chain_head = *self.chain_head.latest_synced.read().await; - if let Err(e) = quarantine::verify_passed( - &self.db, - chain_head, - advance, - self.cfg.canonical_quarantine, - start_block_hash, - ) { + if advance + .header + .height + .saturating_add(self.cfg.canonical_quarantine as u32) + > chain_head.header.height + { return Ok(Acceptance::Rejected(format!( - "advance {advance} (parent_advanced {parent_advanced}) \ - not yet covered by local view: {e}" + "advance EB {advance} does not pass quarantine against local chain head {chain_head}", ))); } + let parent_advanced = parent_hash + .is_zero() + .then(H256::zero) + .unwrap_or_else(|| self.db.mb_meta(parent_hash).last_advanced_eb); + let start_block_hash = self.db.globals().start_block_hash; match quarantine::is_strict_descendant_of( &self.db, advance, parent_advanced, start_block_hash, ) { - Ok(true) => {} - Ok(false) => { - return Ok(Acceptance::Rejected(format!( - "advance {advance} is not a strict descendant of \ - parent.last_advanced_eb {parent_advanced}" - ))); - } - Err(e) => { + Ok(Acceptance::Accepted(())) => {} + Ok(Acceptance::Rejected(reason)) => { return Ok(Acceptance::Rejected(format!( - "is_strict_descendant_of failed for advance {advance} \ - (parent_advanced {parent_advanced}): {e}" + "advance {advance} is not a strict descendant of parent_advanced {parent_advanced}: {reason}" ))); } + Err(e) => return Err(e), } } // Validate injected txs - let chain_head = *self.chain_head.latest_synced.read().await; let checker = TxValidityChecker::new_for_mb(self.db.clone(), chain_head, parent_hash)?; for tx in payload.iter() { let Operation::Injected(signed) = tx else { @@ -523,7 +523,7 @@ impl EthexeExternalities { async fn wait_for_proposable_content( &self, prev_advanced_eb_hash: H256, - ) -> (Option, Vec) { + ) -> Result<(Option, Vec)> { loop { let chain_head_notified = self.chain_head.notify.notified(); tokio::pin!(chain_head_notified); @@ -531,7 +531,7 @@ impl EthexeExternalities { let advance = self .find_eb_candidate_for_advancing(prev_advanced_eb_hash) - .await; + .await?; let chain_head = *self.chain_head.latest_synced.read().await; let Some(mempool) = self.mempool.as_ref() else { @@ -540,7 +540,7 @@ impl EthexeExternalities { let injected_txs = mempool.fetch(chain_head).await; if advance.is_some() || !injected_txs.is_empty() { - return (advance, injected_txs); + return Ok((advance, injected_txs)); } tokio::select! { @@ -552,39 +552,38 @@ impl EthexeExternalities { } // Candidate EB must be anchored in the quarantine and a strict descendant of the previously advanced EB. - async fn find_eb_candidate_for_advancing(&self, prev_advanced_eb_hash: H256) -> Option { - let head = *self.chain_head.latest_synced.read().await; + async fn find_eb_candidate_for_advancing(&self, parent_advance: H256) -> Result> { + let chain_head = *self.chain_head.latest_synced.read().await; let start = self.db.globals().start_block_hash; let total_depth = self.cfg.canonical_quarantine as u32 + self.cfg.post_quarantine_delay; - let candidate = match quarantine::anchor(&self.db, head, total_depth, start) { + + let candidate = match quarantine::anchor(&self.db, chain_head, total_depth, start) { Ok(Some(c)) => c, - Ok(None) => return None, - Err(e) => { - warn!(error = %e, "anchor lookup failed; skipping advance"); - return None; + Ok(None) => { + trace!("anchor lookup reached start block; skipping advance"); + return Ok(None); } + Err(e) => return Err(anyhow!("quarantine anchor lookup failed: {e}")), }; - if candidate == prev_advanced_eb_hash { - return None; - } - match quarantine::is_strict_descendant_of(&self.db, candidate, prev_advanced_eb_hash, start) - { - Ok(true) => Some(candidate), - Ok(false) => None, - Err(e) => { - error!( - error = %e, + + match quarantine::is_strict_descendant_of(&self.db, candidate, parent_advance, start) { + Ok(Acceptance::Accepted(())) => Ok(Some(candidate.hash)), + Ok(Acceptance::Rejected(reason)) => { + warn!( + reason = %reason, candidate = %candidate, - parent_advanced = %prev_advanced_eb_hash, + parent_advanced = %parent_advance, "quarantine-passed EB is not a canonical descendant of \ parent's last_advanced_eb — skipping AdvanceTillEthereumBlock" ); - None + Ok(None) } + Err(e) => Err(e).context("quarantine descendant check failed"), } } } +#[cfg(feature = "disable-tests")] #[cfg(test)] mod tests { use super::*; @@ -995,11 +994,14 @@ mod tests { #[async_trait::async_trait] impl Mempool for ForgetTracker { - fn insert(&self, _tx: SignedInjectedTransaction) -> crate::mempool::TxInsertionStatus { + async fn insert( + &self, + _tx: SignedInjectedTransaction, + ) -> crate::mempool::TxInsertionStatus { crate::mempool::TxInsertionStatus::Inserted } - fn set_chain_head(&self, _head: SimpleBlockData) -> Vec { + async fn set_chain_head(&self, _head: SimpleBlockData) -> Vec { Vec::new() } diff --git a/ethexe/malachite/service/src/mempool.rs b/ethexe/malachite/service/src/mempool.rs index ddefa48bb24..90052b61b21 100644 --- a/ethexe/malachite/service/src/mempool.rs +++ b/ethexe/malachite/service/src/mempool.rs @@ -30,11 +30,6 @@ //! resolve `reference_block` into heights and to walk ancestor links; //! all DB reads are synchronous and cheap (RocksDB point lookups). -use std::{ - collections::{HashMap, HashSet}, - sync::{Arc, Mutex}, -}; - use async_trait::async_trait; use ethexe_common::{ HashOf, SimpleBlockData, @@ -46,7 +41,11 @@ use ethexe_common::{ }; use ethexe_db::Database; use gprimitives::H256; -use tokio::sync::Notify; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use tokio::sync::{Notify, RwLock}; use tracing::{info, trace}; /// Outcome of [`Mempool::insert`]. Splits into two groups: @@ -115,16 +114,11 @@ impl From for InjectedTransactionAcceptance { /// `forget` runs after MB finalization and dedups within `VALIDITY_WINDOW`. #[async_trait] pub trait Mempool: Send + Sync + 'static { - /// Every pool-policy outcome — including the rejecting ones — is a - /// [`TxInsertionStatus`] value. The method is infallible: invariant - /// violations inside the implementation panic (e.g. a poisoned mutex) - /// rather than surface as an error variant. - fn insert(&self, tx: SignedInjectedTransaction) -> TxInsertionStatus; + /// Attempt to insert a new transaction into the pool. The tx's hash is + async fn insert(&self, tx: SignedInjectedTransaction) -> TxInsertionStatus; - /// Drives validity-window GC. - /// Returns the purged injected transactions. - #[must_use] - fn set_chain_head(&self, head: SimpleBlockData) -> Vec; + /// Notify the pool of a new chain head. The pool uses this to evict expired + async fn set_chain_head(&self, head: SimpleBlockData) -> Vec; /// Txs whose `reference_block` is an ancestor of `head`. async fn fetch(&self, head: SimpleBlockData) -> Vec; @@ -146,11 +140,11 @@ pub(crate) struct EmptyMempool; #[cfg(test)] #[async_trait] impl Mempool for EmptyMempool { - fn insert(&self, _tx: SignedInjectedTransaction) -> TxInsertionStatus { + async fn insert(&self, _tx: SignedInjectedTransaction) -> TxInsertionStatus { TxInsertionStatus::Inserted } - fn set_chain_head(&self, _head: SimpleBlockData) -> Vec { + async fn set_chain_head(&self, _head: SimpleBlockData) -> Vec { Vec::new() } @@ -185,10 +179,10 @@ struct Inner { #[derive(Debug)] pub struct InjectedTxMempool { - inner: Mutex, + inner: RwLock, db: Database, capacity: usize, - /// Raised on insert; awaited by the producer in `wait_for_new_tx`. + /// Notification about new transactions in `wait_for_new_tx`. new_tx_notify: Arc, } @@ -199,19 +193,19 @@ impl InjectedTxMempool { pub fn with_capacity(db: Database, capacity: usize) -> Self { Self { - inner: Mutex::new(Inner::default()), + inner: RwLock::new(Inner::default()), db, capacity, new_tx_notify: Arc::new(Notify::new()), } } - pub fn len(&self) -> usize { - self.inner.lock().expect("poisoned mempool").pool.len() + pub async fn len(&self) -> usize { + self.inner.read().await.pool.len() } - pub fn is_empty(&self) -> bool { - self.inner.lock().expect("poisoned mempool").pool.is_empty() + pub async fn is_empty(&self) -> bool { + self.inner.read().await.pool.is_empty() } /// Resolve `reference_block` to its canonical height via the DB. @@ -231,14 +225,14 @@ impl InjectedTxMempool { } /// Set of ancestors of `head` within `VALIDITY_WINDOW` steps. - fn recent_ancestors(&self, head: &SimpleBlockData) -> HashSet { + fn recent_ancestors(&self, head_eb: &SimpleBlockData) -> HashSet { let start_fence = self.start_block_hash(); let mut ancestors = HashSet::with_capacity(VALIDITY_WINDOW as usize + 1); - ancestors.insert(head.hash); + ancestors.insert(head_eb.hash); - let mut current = head.hash; - let mut parent = head.header.parent_hash; + let mut current = head_eb.hash; + let mut parent = head_eb.header.parent_hash; for _ in 0..VALIDITY_WINDOW { if current == start_fence || parent == H256::zero() { break; @@ -303,7 +297,7 @@ impl InjectedTxMempool { #[async_trait] impl Mempool for InjectedTxMempool { - fn insert(&self, tx: SignedInjectedTransaction) -> TxInsertionStatus { + async fn insert(&self, tx: SignedInjectedTransaction) -> TxInsertionStatus { let tx_data = tx.data(); let tx_hash = tx_data.to_hash(); let ref_block = tx_data.reference_block; @@ -321,7 +315,7 @@ impl Mempool for InjectedTxMempool { return TxInsertionStatus::NonZeroValue; } - let inner = self.inner.lock().expect("poisoned mempool"); + let inner = self.inner.read().await; if inner.seen.contains_key(&tx_hash) { info!(%tx_hash, "mempool: idempotent no-op — hash already committed within validity window"); @@ -370,7 +364,7 @@ impl Mempool for InjectedTxMempool { // writes converge on the same byte content. self.db.set_injected_transaction(tx.clone()); - let mut inner = self.inner.lock().expect("poisoned mempool"); + let mut inner = self.inner.write().await; // Recheck dedup / capacity after the lock-free window. if inner.seen.contains_key(&tx_hash) { @@ -400,8 +394,8 @@ impl Mempool for InjectedTxMempool { TxInsertionStatus::Inserted } - fn set_chain_head(&self, head: SimpleBlockData) -> Vec { - let mut inner = self.inner.lock().expect("poisoned mempool"); + async fn set_chain_head(&self, head: SimpleBlockData) -> Vec { + let mut inner = self.inner.write().await; let h = head.header.height; if inner.latest_head_height == Some(h) { // Same height re-sent — nothing to GC beyond what we @@ -415,7 +409,7 @@ impl Mempool for InjectedTxMempool { async fn fetch(&self, head: SimpleBlockData) -> Vec { let ancestors = self.recent_ancestors(&head); - let inner = self.inner.lock().expect("poisoned mempool"); + let inner = self.inner.read().await; let pool_len = inner.pool.len(); let result: Vec<_> = inner .pool @@ -435,7 +429,7 @@ impl Mempool for InjectedTxMempool { } async fn forget(&self, committed: &[SignedInjectedTransaction]) { - let mut inner = self.inner.lock().expect("poisoned mempool"); + let mut inner = self.inner.write().await; for tx in committed { let tx_hash = tx.data().to_hash(); inner.pool.remove(&tx_hash); @@ -456,6 +450,7 @@ impl Mempool for InjectedTxMempool { } } +#[cfg(feature = "disable-tests")] #[cfg(test)] mod tests { use super::*; diff --git a/ethexe/malachite/service/src/quarantine.rs b/ethexe/malachite/service/src/quarantine.rs index da96eec6a3a..49ee1a44013 100644 --- a/ethexe/malachite/service/src/quarantine.rs +++ b/ethexe/malachite/service/src/quarantine.rs @@ -23,141 +23,87 @@ //! - *"quarantine-passed"* means the block has ≥ `canonical_quarantine` //! canonical descendants on top. -use anyhow::{Result, anyhow}; -use ethexe_common::{SimpleBlockData, db::OnChainStorageRO}; -use ethexe_db::Database; +use anyhow::{Context as _, Result, anyhow}; +use ethexe_common::{Acceptance, SimpleBlockData, db::OnChainStorageRO}; use gprimitives::H256; -/// Cap on parent walks when verifying a peer's `AdvanceTillEthereumBlock`. -const VERIFY_LOOKBACK_SLACK: u32 = 100_000; - -/// Youngest EB that has cleared quarantine. `Ok(None)` if the local chain is -/// too short, `Err` on missing parent header. +/// Youngest EB that has cleared the `depth` quarantine against the local head. +/// Returns `Ok(None)` if no blocks passed the depth check since `start_eb`. /// -/// `depth` is taken as `u32` to accommodate the proposer-side -/// `canonical_quarantine + post_quarantine_delay` sum, which can exceed -/// `u8::MAX`. `verify_passed` still takes `u8` because the protocol -/// invariant validators enforce is anchored on `canonical_quarantine` -/// only — the extra slack is a proposer-side hint. +/// # Caller guarantees +/// - `head` is synced block +/// - `start_eb_hash` is hash of global start block (DB genesis or fast-sync pivot) pub fn anchor( - db: &Database, + db: &impl OnChainStorageRO, head: SimpleBlockData, depth: u32, - start_block_hash: H256, -) -> Result> { - let mut current = head.hash; - let mut header = head.header; - + start_eb: H256, +) -> Result> { + let mut cursor = head; for _ in 0..depth { - if current == start_block_hash { + if cursor.hash == start_eb { return Ok(None); } - let parent = header.parent_hash; - header = db - .block_header(parent) - .ok_or_else(|| anyhow!("quarantine anchor: missing parent header for {parent}"))?; - current = parent; + cursor = db + .block_simple_data(cursor.header.parent_hash) + .ok_or_else(|| anyhow!("quarantine anchor: missing header for {cursor}"))?; } - Ok(Some(current)) + Ok(Some(cursor)) } -/// `candidate` must be a canonical ancestor of `head` at depth ≥ `canonical_quarantine`. -/// `Err` on still-quarantined / not-found / missing-parent. -pub fn verify_passed( - db: &Database, - head: SimpleBlockData, - candidate: H256, - canonical_quarantine: u8, - start_block_hash: H256, -) -> Result<()> { - let canonical_quarantine = canonical_quarantine as u32; - let max_steps = canonical_quarantine.saturating_add(VERIFY_LOOKBACK_SLACK); +/// Check whether `candidate` strictly descends from `ancestor` +/// +/// # Caller guarantees +/// - `candidate` is synced block +/// - `ancestor` is hash of synced block or zero (pre-genesis sentinel) +/// - `start_eb_hash` is hash of global start block (DB genesis or fast-sync pivot) +pub fn is_strict_descendant_of( + db: &impl OnChainStorageRO, + candidate: SimpleBlockData, + ancestor_hash: H256, + start_eb_hash: H256, +) -> Result> { + if ancestor_hash == H256::zero() { + // Special case: all blocks descend from the zero hash (pre-genesis sentinel). + return Ok(Acceptance::Accepted(())); + } - let mut current = head.hash; - let mut header = head.header; + let ancestor = db.block_simple_data(ancestor_hash).with_context(|| { + anyhow!("descendant check: missing header for ancestor {ancestor_hash}") + })?; - for depth in 0..=max_steps { - if current == candidate { - return if depth >= canonical_quarantine { - Ok(()) - } else { - Err(anyhow!( - "EB {candidate} is only {depth} block(s) behind head, \ - needs ≥ {canonical_quarantine}" - )) - }; + let Some(depth) = candidate.header.height.checked_sub(ancestor.header.height) else { + return Ok(Acceptance::Rejected(format!( + "candidate {candidate} height is not greater than ancestor {ancestor}" + ))); + }; + + let mut cursor = candidate; + for _ in 0..depth { + let parent_hash = cursor.header.parent_hash; + if parent_hash == ancestor.hash { + // Found the ancestor + return Ok(Acceptance::Accepted(())); } - if current == start_block_hash { - return Err(anyhow!( - "EB {candidate} is not a canonical ancestor of local chain head \ - (walk reached start_block at depth {depth})" - )); + if parent_hash == start_eb_hash { + return Ok(Acceptance::Rejected(format!( + "ancestor {ancestor} not found within candidate's ancestry (walk reached start EB)" + ))); } - let parent = header.parent_hash; - header = db.block_header(parent).ok_or_else(|| { - anyhow!("quarantine verify: missing parent header for {parent} at depth {depth}") + cursor = db.block_simple_data(parent_hash).with_context(|| { + anyhow!("descendant check: missing header for parent {parent_hash}") })?; - current = parent; - } - - Err(anyhow!( - "EB {candidate} not found within {max_steps} ancestors of local chain head" - )) -} - -/// `candidate` strictly descends from `ancestor` (depth ≥ 1). `H256::zero()` = -/// pre-genesis sentinel; equal hashes return `Ok(false)`. -pub fn is_strict_descendant_of( - db: &Database, - candidate: H256, - ancestor: H256, - start_block_hash: H256, -) -> Result { - if ancestor.is_zero() { - return Ok(true); - } - if candidate == ancestor { - return Ok(false); - } - - let max_steps = VERIFY_LOOKBACK_SLACK; - let mut current = candidate; - let mut header = db - .block_header(current) - .ok_or_else(|| anyhow!("descendant check: missing header for candidate {candidate}"))?; - - for _ in 0..max_steps { - let parent = header.parent_hash; - if parent == ancestor { - return Ok(true); - } - if parent == H256::zero() { - return Err(anyhow!( - "descendant check: ancestor {ancestor} not in canonical ancestry of \ - candidate {candidate} — walk reached genesis" - )); - } - if current == start_block_hash { - return Err(anyhow!( - "descendant check: ancestor {ancestor} not found before start_block fence \ - starting from candidate {candidate}" - )); - } - header = db - .block_header(parent) - .ok_or_else(|| anyhow!("descendant check: missing parent header for {parent}"))?; - current = parent; } - Err(anyhow!( - "descendant check: ancestor {ancestor} not found within {max_steps} ancestors \ - of candidate {candidate}" - )) + Ok(Acceptance::Rejected(format!( + "candidate {candidate} does not descend from ancestor {ancestor}" + ))) } +#[cfg(feature = "disable-tests")] #[cfg(test)] mod tests { use super::*; diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index dd616b2be51..320f885a9c8 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -20,6 +20,7 @@ use crate::{ MalachiteEvent, Mempool, ValidatorEntry, externalities::EthexeExternalities, mempool::TxInsertionStatus, types::ChainHead, }; +use alloy::providers::BoxedFut; use anyhow::Result; use ethexe_common::{ Address, SimpleBlockData, @@ -27,11 +28,11 @@ use ethexe_common::{ injected::SignedInjectedTransaction, }; use ethexe_malachite_core::MalachiteCore; -use futures::{Stream, stream::FusedStream}; +use futures::{FutureExt, Stream, stream::FusedStream}; use gprimitives::H256; use gsigner::schemes::secp256k1::PublicKey; use std::{ - collections::HashMap, + collections::{HashMap, VecDeque}, pin::Pin, sync::Arc, task::{Context, Poll}, @@ -56,19 +57,23 @@ impl Drop for MalachiteService { } impl MalachiteService { - pub fn receive_injected_transaction(&self, tx: SignedInjectedTransaction) -> TxInsertionStatus { - self.mempool - .as_ref() - .map(|pool| pool.insert(tx)) - .unwrap_or(TxInsertionStatus::PoolFull) + pub async fn receive_injected_transaction( + &self, + tx: SignedInjectedTransaction, + ) -> TxInsertionStatus { + if let Some(pool) = self.mempool.as_ref() { + pool.insert(tx).await + } else { + TxInsertionStatus::PoolFull + } } - pub async fn receive_new_chain_head(&mut self, head: SimpleBlockData) { + pub async fn receive_new_eb(&mut self, eb: SimpleBlockData) { let mut current = self.chain_head.latest.write().await; // Filter the new head against the current one and update if it's strictly higher - if head.header.height > current.header.height { - *current = head; + if eb.header.height > current.header.height { + *current = eb; } } @@ -95,7 +100,7 @@ impl MalachiteService { self.maybe_rotate_validators_for_era(chain_head); if let Some(pool) = self.mempool.as_ref() { - let purged_txs = pool.set_chain_head(chain_head); + let purged_txs = pool.set_chain_head(chain_head).await; if !purged_txs.is_empty() { let event = MalachiteEvent::PurgedTransactions { eb_hash: chain_head.hash, @@ -115,6 +120,12 @@ impl MalachiteService { self.externalities.drain_pending_events().await } + pub async fn shutdown(mut self) { + if let Some(inner) = self.inner.take() { + inner.shutdown().await; + } + } + /// Push the on-chain validators for `head`'s era into the engine, /// if the era moved. Skips on missing DB data or unknown pub keys /// (wait-and-retry: the next `BlockSynced` re-evaluates). @@ -178,18 +189,15 @@ impl MalachiteService { "rotated malachite validator set to era's on-chain quorum" ); } - - pub async fn shutdown(mut self) { - if let Some(inner) = self.inner.take() { - inner.shutdown().await; - } - } } impl Stream for MalachiteService { type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // IMPORTANT: MalachiteService async methods (like receive_new_eb and other) + // are safe to call only because we do not lock any data from self in this method implementation. + if let Some(inner) = self.inner.as_mut() { match Pin::new(&mut *inner).poll_next(cx) { Poll::Ready(Some(e)) => return Poll::Ready(Some(Err(e))), @@ -199,6 +207,7 @@ impl Stream for MalachiteService { Poll::Pending => {} } } + self.events_rx.poll_recv(cx) } } diff --git a/ethexe/malachite/service/src/types.rs b/ethexe/malachite/service/src/types.rs index 7405a7ada49..469b11855fd 100644 --- a/ethexe/malachite/service/src/types.rs +++ b/ethexe/malachite/service/src/types.rs @@ -1,4 +1,5 @@ -use ethexe_common::SimpleBlockData; +use ethexe_common::{SimpleBlockData, injected::SignedInjectedTransaction}; +use gprimitives::H256; use tokio::sync::{Notify, RwLock}; pub struct ChainHead { diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index 9d925733703..47bf2ee5878 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -16,6 +16,8 @@ //! `globals.latest_finalized_mb_hash` is gap-free across the //! restart boundary, and the latest pointer never rewinds. +#![cfg(feature = "disable-tests")] + use std::{path::Path, time::Duration}; use async_trait::async_trait; @@ -145,7 +147,7 @@ fn build_config( /// and immediately mark it synced (the test seeds all headers/events /// upfront, so "observed" and "synced" coincide here). async fn feed_head(service: &mut MalachiteService, head: SimpleBlockData) { - service.receive_new_chain_head(head).await; + service.receive_new_eb(head).await; service.receive_eb_synced(head.hash).await; } diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 7182216c289..5ecf8a427d4 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -713,7 +713,7 @@ impl Service { c.receive_new_chain_head(block)?; } - malachite.receive_new_chain_head(block).await; + malachite.receive_new_eb(block).await; } ObserverEvent::BlockSynced(block_hash) => { log::info!("Ethereum block synced: {block_hash}"); @@ -835,8 +835,10 @@ impl Service { transaction, channel, } => { - let acceptance = - malachite.receive_injected_transaction(*transaction).into(); + let acceptance = malachite + .receive_injected_transaction(*transaction) + .await + .into(); if let Err(err) = channel.send(acceptance) { tracing::error!( ?err, @@ -880,8 +882,9 @@ impl Service { transaction, response_sender, } => { - let status = - malachite.receive_injected_transaction(transaction.clone()); + let status = malachite + .receive_injected_transaction(transaction.clone()) + .await; let local_acceptance = InjectedTransactionAcceptance::from(status); match network.as_mut() { @@ -967,10 +970,7 @@ impl Service { mb_hash = %mb_hash, "Malachite: BlockProposal", ); - // Validators are interested in this MB's - // promises so they can gossip them; the - // service's `PromiseEmissionMode` can still - // force the policy to `Enabled` regardless. + compute.compute_mb(mb_hash, ethexe_common::PromisePolicy::Enabled); } MalachiteEvent::BlockFinalized { @@ -984,6 +984,8 @@ impl Service { sigs = cert.signatures.len(), "Malachite: BlockFinalized", ); + + // +_+_+ ??? whether we really need this? // Non-proposer nodes (validators that didn't propose // this height + every full/RPC node) first see the MB // here. Trigger compute so the body — including any From d5d3c41a66cea6f6b7390b659ae3d550e36eb62b Mon Sep 17 00:00:00 2001 From: Grisha Sobol Date: Thu, 11 Jun 2026 14:00:21 +0200 Subject: [PATCH 33/72] test(vara.eth/malachite): re-enable gated tests after errors-handling rework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop all `disable-tests` cfg gates. Port mempool tests to the async Mempool API (proptests drive it via block_on), quarantine tests to the SimpleBlockData/Acceptance signatures (drop `verify_passed` proptest — the depth check now lives in validate_block_above), and flip the unresolved-ref_block test to assert the now-implemented purge. Co-Authored-By: Claude Opus 4.7 --- ethexe/malachite/service/src/externalities.rs | 32 +-- ethexe/malachite/service/src/mempool.rs | 205 ++++++++++-------- ethexe/malachite/service/src/quarantine.rs | 137 +++++------- ethexe/malachite/service/src/service.rs | 5 +- ethexe/malachite/service/src/types.rs | 3 +- .../service/tests/restart_resilience.rs | 6 +- 6 files changed, 187 insertions(+), 201 deletions(-) diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index 748bf8ca312..8ddb65fb315 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -583,7 +583,6 @@ impl EthexeExternalities { } } -#[cfg(feature = "disable-tests")] #[cfg(test)] mod tests { use super::*; @@ -1249,7 +1248,7 @@ mod tests { let mempool = Arc::new(crate::InjectedTxMempool::new(db.clone())); // Drive validity-window GC. - let _ = mempool.set_chain_head(head); + let _ = mempool.set_chain_head(head).await; let pk = ethexe_common::PrivateKey::random(); let valid = signed_injected_tx(&pk, dest, chain.blocks[9].hash, 0); @@ -1265,12 +1264,12 @@ mod tests { ) .unwrap(); - mempool.insert(valid.clone()); + mempool.insert(valid.clone()).await; assert_eq!( - mempool.insert(value_tx.clone()), + mempool.insert(value_tx.clone()).await, crate::mempool::TxInsertionStatus::NonZeroValue, ); - assert_eq!(mempool.len(), 1); + assert_eq!(mempool.len().await, 1); let (ext, _rx) = make_externalities_with_pool(db, mempool); set_head(&ext, head).await; @@ -1329,19 +1328,21 @@ mod tests { let head = chain.blocks[10].to_simple(); let mempool = Arc::new(crate::InjectedTxMempool::new(db.clone())); - let _ = mempool.set_chain_head(head); + let _ = mempool.set_chain_head(head).await; let pk = ethexe_common::PrivateKey::random(); // Push 50 txs targeting the upper half of destinations (the ones // NOT pre-touched by EB events). let push_start = MAX_TOUCHED_PROGRAMS_PER_MB / 2 + 1; let push_end = MAX_TOUCHED_PROGRAMS_PER_MB + 1; for i in push_start..push_end { - mempool.insert(signed_injected_tx( - &pk, - ActorId::from(i as u64), - chain.blocks[9].hash, - i as u8, - )); + mempool + .insert(signed_injected_tx( + &pk, + ActorId::from(i as u64), + chain.blocks[9].hash, + i as u8, + )) + .await; } let (ext, _rx) = make_externalities_with_pool(db.clone(), mempool); @@ -1466,7 +1467,7 @@ mod tests { db.globals_mutate(|g| g.latest_computed_mb_hash = parent_mb); let mempool = Arc::new(crate::InjectedTxMempool::new(db.clone())); - let _ = mempool.set_chain_head(head); + let _ = mempool.set_chain_head(head).await; let pk = ethexe_common::PrivateKey::random(); // Each tx carries the maximum-size payload; the pool is loaded // with enough of them that two fit but three don't. @@ -1484,9 +1485,9 @@ mod tests { }, ) .unwrap(); - mempool.insert(tx); + mempool.insert(tx).await; } - assert_eq!(mempool.len(), 3); + assert_eq!(mempool.len().await, 3); let (ext, _rx) = make_externalities_with_pool(db.clone(), mempool); set_head(&ext, head).await; @@ -1895,6 +1896,7 @@ mod tests { let candidate = ext .find_eb_candidate_for_advancing(H256::zero()) .await + .unwrap() .expect("must surface a candidate — chain is deep enough"); // Walk back `2 + 3 = 5` parents from head; that's the expected // anchor. diff --git a/ethexe/malachite/service/src/mempool.rs b/ethexe/malachite/service/src/mempool.rs index 90052b61b21..ef96830b46b 100644 --- a/ethexe/malachite/service/src/mempool.rs +++ b/ethexe/malachite/service/src/mempool.rs @@ -450,7 +450,6 @@ impl Mempool for InjectedTxMempool { } } -#[cfg(feature = "disable-tests")] #[cfg(test)] mod tests { use super::*; @@ -495,8 +494,8 @@ mod tests { } } - #[test] - fn insert_rejects_non_zero_value_before_pool_state_checks() { + #[tokio::test] + async fn insert_rejects_non_zero_value_before_pool_state_checks() { // Verifies NonZeroValue fires *before* the pool is consulted — // a tx with value != 0 must never reach the seen / duplicate / // capacity gates. We seed the pool to capacity first to make @@ -507,7 +506,8 @@ mod tests { let pk = PrivateKey::random(); // Fill to capacity with a valid tx so PoolFull would normally fire. - pool.insert(signed_tx(&pk, ActorId::zero(), chain[1].hash, 0)); + pool.insert(signed_tx(&pk, ActorId::zero(), chain[1].hash, 0)) + .await; let value_tx = SignedMessage::create( pk.clone(), @@ -521,60 +521,64 @@ mod tests { ) .unwrap(); - assert_eq!(pool.insert(value_tx), TxInsertionStatus::NonZeroValue,); - assert_eq!(pool.len(), 1, "non-zero-value tx must not enter the pool"); + assert_eq!(pool.insert(value_tx).await, TxInsertionStatus::NonZeroValue); + assert_eq!( + pool.len().await, + 1, + "non-zero-value tx must not enter the pool" + ); } /// Fresh insert that passes every gate must return `Inserted`. - #[test] - fn insert_returns_inserted_for_fresh_tx() { + #[tokio::test] + async fn insert_returns_inserted_for_fresh_tx() { let db = Database::memory(); let chain = linear_chain(&db, 2); let pool = InjectedTxMempool::new(db); let pk = PrivateKey::random(); let tx = signed_tx(&pk, ActorId::zero(), chain[1].hash, 0); - assert_eq!(pool.insert(tx), TxInsertionStatus::Inserted); - assert_eq!(pool.len(), 1); + assert_eq!(pool.insert(tx).await, TxInsertionStatus::Inserted); + assert_eq!(pool.len().await, 1); } /// Same tx inserted twice — second insert hits the pool table and /// returns `AlreadyInPool` without bumping the size. - #[test] - fn insert_returns_already_in_pool_for_duplicate() { + #[tokio::test] + async fn insert_returns_already_in_pool_for_duplicate() { let db = Database::memory(); let chain = linear_chain(&db, 2); let pool = InjectedTxMempool::new(db); let pk = PrivateKey::random(); let tx = signed_tx(&pk, ActorId::zero(), chain[1].hash, 5); - assert_eq!(pool.insert(tx.clone()), TxInsertionStatus::Inserted); - assert_eq!(pool.insert(tx), TxInsertionStatus::AlreadyInPool,); - assert_eq!(pool.len(), 1); + assert_eq!(pool.insert(tx.clone()).await, TxInsertionStatus::Inserted); + assert_eq!(pool.insert(tx).await, TxInsertionStatus::AlreadyInPool); + assert_eq!(pool.len().await, 1); } /// After `forget`, re-inserting the same tx hits the seen-hash table /// and returns `AlreadyIncluded`. - #[test] - fn insert_returns_already_included_for_committed_tx() { + #[tokio::test] + async fn insert_returns_already_included_for_committed_tx() { let db = Database::memory(); let chain = linear_chain(&db, 2); let pool = InjectedTxMempool::new(db); let pk = PrivateKey::random(); let tx = signed_tx(&pk, ActorId::zero(), chain[1].hash, 11); - pool.insert(tx.clone()); - futures::executor::block_on(pool.forget(std::slice::from_ref(&tx))); - assert_eq!(pool.len(), 0); + pool.insert(tx.clone()).await; + pool.forget(std::slice::from_ref(&tx)).await; + assert_eq!(pool.len().await, 0); - assert_eq!(pool.insert(tx), TxInsertionStatus::AlreadyIncluded,); - assert_eq!(pool.len(), 0); + assert_eq!(pool.insert(tx).await, TxInsertionStatus::AlreadyIncluded); + assert_eq!(pool.len().await, 0); } /// `ExpiredRefBlock` fires once `set_chain_head` has advanced past /// `ref_block_height + VALIDITY_WINDOW` and the tx is brand new. - #[test] - fn insert_returns_expired_ref_block() { + #[tokio::test] + async fn insert_returns_expired_ref_block() { let db = Database::memory(); let chain = linear_chain(&db, (VALIDITY_WINDOW as usize) + 5); let pool = InjectedTxMempool::new(db); @@ -582,11 +586,11 @@ mod tests { // Advance head so block 1 is past the validity window. let head_idx = (VALIDITY_WINDOW as usize) + 1; - let _ = pool.set_chain_head(chain[head_idx]); + let _ = pool.set_chain_head(chain[head_idx]).await; let tx = signed_tx(&pk, ActorId::zero(), chain[1].hash, 0); - assert_eq!(pool.insert(tx), TxInsertionStatus::ExpiredRefBlock,); - assert_eq!(pool.len(), 0); + assert_eq!(pool.insert(tx).await, TxInsertionStatus::ExpiredRefBlock); + assert_eq!(pool.len().await, 0); } /// Persist a synthetic linear chain of length `len` into the DB. @@ -633,18 +637,18 @@ mod tests { .unwrap() } - #[test] - fn insert_unknown_ref_block_is_accepted() { + #[tokio::test] + async fn insert_unknown_ref_block_is_accepted() { let db = Database::memory(); let pool = InjectedTxMempool::new(db); let pk = PrivateKey::random(); let tx = signed_tx(&pk, ActorId::zero(), H256::random(), 1); - pool.insert(tx); - assert_eq!(pool.len(), 1); + pool.insert(tx).await; + assert_eq!(pool.len().await, 1); } - #[test] - fn insert_then_fetch_round_trip() { + #[tokio::test] + async fn insert_then_fetch_round_trip() { let db = Database::memory(); let chain = linear_chain(&db, 3); let pool = InjectedTxMempool::new(db); @@ -653,35 +657,42 @@ mod tests { let tx = signed_tx(&pk, ActorId::zero(), chain[2].hash, 1); let tx_hash = tx.data().to_hash(); - pool.insert(tx.clone()); - assert_eq!(pool.len(), 1); + pool.insert(tx.clone()).await; + assert_eq!(pool.len().await, 1); // The pool fetches when ref_block is on the canonical chain // of the head we hand it. let head = chain[2]; - let fetched = futures::executor::block_on(pool.fetch(head)); + let fetched = pool.fetch(head).await; assert_eq!(fetched.len(), 1); assert_eq!(fetched[0].data().to_hash(), tx_hash); } - #[test] - fn capacity_limit_blocks_further_inserts() { + #[tokio::test] + async fn capacity_limit_blocks_further_inserts() { let db = Database::memory(); let chain = linear_chain(&db, 2); let pool = InjectedTxMempool::with_capacity(db, 2); let pk = PrivateKey::random(); - pool.insert(signed_tx(&pk, ActorId::zero(), chain[1].hash, 0)); - pool.insert(signed_tx(&pk, ActorId::zero(), chain[1].hash, 1)); + pool.insert(signed_tx(&pk, ActorId::zero(), chain[1].hash, 0)) + .await; + pool.insert(signed_tx(&pk, ActorId::zero(), chain[1].hash, 1)) + .await; assert_eq!( - pool.insert(signed_tx(&pk, ActorId::zero(), chain[1].hash, 2)), + pool.insert(signed_tx(&pk, ActorId::zero(), chain[1].hash, 2)) + .await, TxInsertionStatus::PoolFull, ); - assert_eq!(pool.len(), 2, "third insert must hit the capacity cap"); + assert_eq!( + pool.len().await, + 2, + "third insert must hit the capacity cap" + ); } - #[test] - fn pool_retains_unresolved_ref_block_indefinitely() { + #[tokio::test] + async fn unresolved_ref_block_txs_purged_on_head_advance() { let db = Database::memory(); // Use a real chain so set_chain_head can drive `head_height` // forward beyond `VALIDITY_WINDOW`. @@ -692,28 +703,27 @@ mod tests { // 100 txs each anchored at a random ref_block NOT in our DB. for salt in 0..100u8 { let bogus_ref_block = H256::random(); - pool.insert(signed_tx(&pk, ActorId::zero(), bogus_ref_block, salt)); + pool.insert(signed_tx(&pk, ActorId::zero(), bogus_ref_block, salt)) + .await; } - assert_eq!(pool.len(), 100); + assert_eq!(pool.len().await, 100); - // Advance head far past any tx's lifetime. + // Advance head far past any tx's lifetime. Txs whose ref_block + // never resolved are purged as `UnknownReferenceBlock` so the + // public RPC `injected_send` can't permanently exhaust capacity. let head_idx = (VALIDITY_WINDOW as usize) + 1; - let _ = pool.set_chain_head(chain[head_idx]); - - // Desired behaviour: txs whose ref_block never resolved AND - // whose insert is older than VALIDITY_WINDOW should be evicted - // (mirroring the `seen` retain policy). Currently they all - // stay — capacity is permanently exhausted. + let purged = pool.set_chain_head(chain[head_idx]).await; + assert_eq!(pool.len().await, 0, "unresolved-ref_block txs must purge"); + assert_eq!(purged.len(), 100); assert!( - pool.len() < 100, - "pool retains all {} unresolved-ref_block txs after head advanced past WINDOW — \ - public RPC `injected_send` can permanently exhaust capacity", - pool.len(), + purged + .iter() + .all(|p| matches!(p.reason, TransactionPurgedReason::UnknownReferenceBlock)) ); } - #[test] - fn set_chain_head_purges_expired() { + #[tokio::test] + async fn set_chain_head_purges_expired() { let db = Database::memory(); // Build a chain long enough that `head_height - // VALIDITY_WINDOW` passes some block we'll insert against. @@ -723,41 +733,45 @@ mod tests { let pk = PrivateKey::random(); // tx anchored at block 1 — height 1 let tx = signed_tx(&pk, ActorId::zero(), chain[1].hash, 0); - pool.insert(tx); - assert_eq!(pool.len(), 1); + pool.insert(tx).await; + assert_eq!(pool.len().await, 1); // Advance head far enough that block 1's height is past the // validity window. `is_expired` is `ref_height + WINDOW <= head_height`. let head_idx = (VALIDITY_WINDOW as usize) + 1; - let _ = pool.set_chain_head(chain[head_idx]); + let _ = pool.set_chain_head(chain[head_idx]).await; assert_eq!( - pool.len(), + pool.len().await, 0, "set_chain_head should purge txs whose ref_block aged out" ); } - #[test] - fn forget_moves_committed_to_seen_table() { + #[tokio::test] + async fn forget_moves_committed_to_seen_table() { let db = Database::memory(); let chain = linear_chain(&db, 2); let pool = InjectedTxMempool::new(db); let pk = PrivateKey::random(); let tx = signed_tx(&pk, ActorId::zero(), chain[1].hash, 99); - pool.insert(tx.clone()); - assert_eq!(pool.len(), 1); + pool.insert(tx.clone()).await; + assert_eq!(pool.len().await, 1); - futures::executor::block_on(pool.forget(std::slice::from_ref(&tx))); - assert_eq!(pool.len(), 0); + pool.forget(std::slice::from_ref(&tx)).await; + assert_eq!(pool.len().await, 0); // Re-inserting the same tx is a seen-hash no-op. - assert_eq!(pool.insert(tx), TxInsertionStatus::AlreadyIncluded); - assert_eq!(pool.len(), 0, "forgotten tx must not return to the pool"); + assert_eq!(pool.insert(tx).await, TxInsertionStatus::AlreadyIncluded); + assert_eq!( + pool.len().await, + 0, + "forgotten tx must not return to the pool" + ); } - #[test] - fn fetch_filters_non_canonical_branches() { + #[tokio::test] + async fn fetch_filters_non_canonical_branches() { // Two branches diverging at block 1: // genesis (hash[0]) -> b1 (hash[1]) // \-> b1' (hash[1_alt]) @@ -783,19 +797,19 @@ mod tests { // tx anchored to the ALT branch let tx_alt = signed_tx(&pk, ActorId::zero(), alt_hash, 1); - pool.insert(tx_alt); - assert_eq!(pool.len(), 1); + pool.insert(tx_alt).await; + assert_eq!(pool.len().await, 1); // Fetching for canonical branch (chain[1]) — alt tx must NOT // surface. - let fetched = futures::executor::block_on(pool.fetch(chain[1])); + let fetched = pool.fetch(chain[1]).await; assert!( fetched.is_empty(), "tx on alt branch must not be fetched against canonical head" ); // Pool still holds it for a possible reorg. - assert_eq!(pool.len(), 1); + assert_eq!(pool.len().await, 1); } #[tokio::test(start_paused = true)] @@ -815,7 +829,8 @@ mod tests { tokio::time::sleep(Duration::from_millis(10)).await; let pk = PrivateKey::random(); - pool.insert(signed_tx(&pk, ActorId::zero(), chain[1].hash, 0)); + pool.insert(signed_tx(&pk, ActorId::zero(), chain[1].hash, 0)) + .await; // Waiter should now wake up promptly. tokio::time::timeout(Duration::from_secs(1), waiter) @@ -837,7 +852,7 @@ mod tests { // Seed one accepted insert and consume the resulting permit so // the next `.notified()` re-blocks until the next signal. - pool.insert(tx.clone()); + pool.insert(tx.clone()).await; pool.wait_for_new_tx().await; let waiter = { @@ -850,7 +865,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(10)).await; // Same tx hash — idempotent no-op, no signal sent. - pool.insert(tx); + pool.insert(tx).await; // Waiter must still be pending. tokio::time::sleep(Duration::from_millis(50)).await; @@ -881,6 +896,12 @@ mod tests { use proptest::prelude::*; + /// Proptest bodies are sync — drive the pool's async API with a + /// local executor (tokio sync primitives are runtime-agnostic). + fn bo(fut: F) -> F::Output { + futures::executor::block_on(fut) + } + /// Build a deterministic linear chain in `db` and return the /// blocks oldest-first. `seed` makes hashes predictable across /// proptest cases (same input → same chain). @@ -950,7 +971,7 @@ mod tests { // `AlreadyInPool` / `AlreadyIncluded` / capacity // rejects must not feed `live`, otherwise Forget // would target a different occurrence. - if pool.insert(tx.clone()) == TxInsertionStatus::Inserted { + if bo(pool.insert(tx.clone())) == TxInsertionStatus::Inserted { live.push(tx); } } @@ -958,15 +979,15 @@ mod tests { if !live.is_empty() { let idx = which % live.len(); let victim = live.swap_remove(idx); - futures::executor::block_on(pool.forget(std::slice::from_ref(&victim))); + bo(pool.forget(std::slice::from_ref(&victim))); } } } // Capacity invariant — must hold after every step. prop_assert!( - pool.len() <= cap, + bo(pool.len()) <= cap, "pool.len()={} exceeded capacity {}", - pool.len(), + bo(pool.len()), cap ); } @@ -1004,11 +1025,11 @@ mod tests { // Inserts: alternating canonical-tail and alt anchors. for i in 0..n_txs { let anchor = if i % 2 == 0 { chain[3].hash } else { alt_hash }; - pool.insert(signed_tx(&pk, ActorId::zero(), anchor, i as u8)); + bo(pool.insert(signed_tx(&pk, ActorId::zero(), anchor, i as u8))); } let head = chain[3]; - let fetched = futures::executor::block_on(pool.fetch(head)); + let fetched = bo(pool.fetch(head)); for tx in &fetched { prop_assert_ne!( tx.data().reference_block, alt_hash, @@ -1030,14 +1051,14 @@ mod tests { let pool = InjectedTxMempool::new(db); let pk = PrivateKey::random(); let tx = signed_tx(&pk, ActorId::zero(), chain[1].hash, salt); - pool.insert(tx.clone()); - prop_assert_eq!(pool.len(), 1); - futures::executor::block_on(pool.forget(std::slice::from_ref(&tx))); - prop_assert_eq!(pool.len(), 0); + bo(pool.insert(tx.clone())); + prop_assert_eq!(bo(pool.len()), 1); + bo(pool.forget(std::slice::from_ref(&tx))); + prop_assert_eq!(bo(pool.len()), 0); // Re-insert: idempotent no-op because the hash sits in // the seen-set and `reference_block` hasn't aged out. - pool.insert(tx); - prop_assert_eq!(pool.len(), 0); + bo(pool.insert(tx)); + prop_assert_eq!(bo(pool.len()), 0); } } } diff --git a/ethexe/malachite/service/src/quarantine.rs b/ethexe/malachite/service/src/quarantine.rs index 49ee1a44013..a9de9e69508 100644 --- a/ethexe/malachite/service/src/quarantine.rs +++ b/ethexe/malachite/service/src/quarantine.rs @@ -103,18 +103,15 @@ pub fn is_strict_descendant_of( ))) } -#[cfg(feature = "disable-tests")] #[cfg(test)] mod tests { use super::*; - use ethexe_common::{ - BlockHeader, - db::{BlockMetaStorageRW, OnChainStorageRW}, - }; + use ethexe_common::{BlockHeader, db::OnChainStorageRW}; + use ethexe_db::Database; /// Synthetic linear chain, oldest-first; parent[0] == zero. - fn linear_chain(db: &Database, len: usize) -> Vec { - let mut hashes = Vec::with_capacity(len); + fn linear_chain(db: &Database, len: usize) -> Vec { + let mut blocks = Vec::with_capacity(len); let mut parent = H256::zero(); for i in 0..len { let mut hash_bytes = [0u8; 32]; @@ -123,54 +120,64 @@ mod tests { hash_bytes[1] = (i >> 8) as u8; hash_bytes[2] = i as u8; let hash = H256::from(hash_bytes); - db.set_block_header( - hash, - BlockHeader { - height: i as u32, - timestamp: i as u64, - parent_hash: parent, - }, - ); - db.mutate_block_meta(hash, |_| {}); - hashes.push(hash); + let header = BlockHeader { + height: i as u32, + timestamp: i as u64, + parent_hash: parent, + }; + db.set_block_header(hash, header); + blocks.push(SimpleBlockData { hash, header }); parent = hash; } - hashes + blocks } #[test] fn zero_ancestor_is_always_descendant() { let db = Database::memory(); - let hashes = linear_chain(&db, 3); + let chain = linear_chain(&db, 3); // arbitrary candidate; ancestor = zero (pre-genesis sentinel) - assert!(is_strict_descendant_of(&db, hashes[2], H256::zero(), H256::zero()).unwrap()); + assert!( + is_strict_descendant_of(&db, chain[2], H256::zero(), H256::zero()) + .unwrap() + .is_accepted() + ); } #[test] fn same_block_is_not_strict_descendant() { let db = Database::memory(); - let hashes = linear_chain(&db, 3); - assert!(!is_strict_descendant_of(&db, hashes[1], hashes[1], hashes[0]).unwrap()); + let chain = linear_chain(&db, 3); + assert!( + is_strict_descendant_of(&db, chain[1], chain[1].hash, chain[0].hash) + .unwrap() + .is_rejected() + ); } #[test] - fn proper_ancestor_resolves_to_true() { + fn proper_ancestor_resolves_to_accepted() { let db = Database::memory(); - let hashes = linear_chain(&db, 5); - // hashes[4] should be a strict descendant of hashes[1] + let chain = linear_chain(&db, 5); + // chain[4] should be a strict descendant of chain[1] // through 3 parent steps. - assert!(is_strict_descendant_of(&db, hashes[4], hashes[1], hashes[0]).unwrap()); + assert!( + is_strict_descendant_of(&db, chain[4], chain[1].hash, chain[0].hash) + .unwrap() + .is_accepted() + ); } #[test] fn unrelated_ancestor_errors() { let db = Database::memory(); - let hashes = linear_chain(&db, 5); - // ancestor = a hash that's not in the chain at all + let chain = linear_chain(&db, 5); + // ancestor = a hash that's not in the chain at all — the local + // header lookup fails, which is a local-view error, not a vote. let mut orphan_bytes = [0xFFu8; 32]; orphan_bytes[0] = 0x42; let orphan = H256::from(orphan_bytes); - let res = is_strict_descendant_of(&db, hashes[4], orphan, hashes[0]); + let res = is_strict_descendant_of(&db, chain[4], orphan, chain[0].hash); assert!(res.is_err(), "expected Err for orphan ancestor: {res:?}"); } @@ -185,8 +192,8 @@ mod tests { /// `anchor(head)` walks back exactly `canonical_quarantine` /// steps along the canonical chain, so for any pair - /// (chain_len, q) with `q < chain_len` the returned hash is - /// the block at index `chain_len - 1 - q`. + /// (chain_len, q) with `q < chain_len` the returned block is + /// the one at index `chain_len - 1 - q`. #[test] fn anchor_walks_exactly_q_steps( chain_len in 2usize..32, @@ -195,25 +202,18 @@ mod tests { let q_usize = q as usize; prop_assume!(q_usize < chain_len); let db = Database::memory(); - let hashes = linear_chain(&db, chain_len); - let head = SimpleBlockData { - hash: hashes[chain_len - 1], - header: ethexe_common::BlockHeader { - height: (chain_len - 1) as u32, - timestamp: (chain_len - 1) as u64, - parent_hash: if chain_len >= 2 { hashes[chain_len - 2] } else { H256::zero() }, - }, - }; + let chain = linear_chain(&db, chain_len); + let head = chain[chain_len - 1]; // start_block = genesis (so the fence never trips). - let result = anchor(&db, head, q as u32, hashes[0]).unwrap(); - let expected = hashes[chain_len - 1 - q_usize]; + let result = anchor(&db, head, q as u32, chain[0].hash).unwrap(); + let expected = chain[chain_len - 1 - q_usize]; prop_assert_eq!(result, Some(expected)); } /// `is_strict_descendant_of(c, a)` is the transitive closure /// of "next-block": for any (i, j) on a single chain, with - /// `i > j > 0`, the chain[i] descends from chain[j]; with - /// `i == j`, it does NOT (strictness). + /// `i > j`, chain[i] descends from chain[j]; with `i <= j`, + /// it does NOT (strictness / height check). #[test] fn descendant_relation_matches_chain_indices( chain_len in 2usize..16, @@ -223,50 +223,17 @@ mod tests { prop_assume!(i < chain_len); prop_assume!(j < chain_len); let db = Database::memory(); - let hashes = linear_chain(&db, chain_len); + let chain = linear_chain(&db, chain_len); - let result = is_strict_descendant_of(&db, hashes[i], hashes[j], hashes[0]); + let result = is_strict_descendant_of(&db, chain[i], chain[j].hash, chain[0].hash) + .unwrap(); if i > j { - prop_assert_eq!(result.unwrap(), true); - } else if i == j { - prop_assert_eq!(result.unwrap(), false); - } else { - // i < j → walking back from i never reaches j. - // The walk hits genesis (parent_hash zero) → Err. - prop_assert!(result.is_err()); - } - } - - /// `verify_passed(head, candidate)` succeeds iff `candidate` - /// sits at depth >= q from `head` on the canonical chain. - #[test] - fn verify_passed_matches_depth( - chain_len in 4usize..16, - head_idx in 0usize..16, - cand_idx in 0usize..16, - q in 0u8..6, - ) { - prop_assume!(head_idx < chain_len); - prop_assume!(cand_idx <= head_idx); - let db = Database::memory(); - let hashes = linear_chain(&db, chain_len); - let head_hash = hashes[head_idx]; - let head_height = head_idx as u32; - let head_parent = if head_idx > 0 { hashes[head_idx - 1] } else { H256::zero() }; - let head = SimpleBlockData { - hash: head_hash, - header: ethexe_common::BlockHeader { - height: head_height, - timestamp: head_idx as u64, - parent_hash: head_parent, - }, - }; - let depth = head_idx - cand_idx; - let result = verify_passed(&db, head, hashes[cand_idx], q, hashes[0]); - if depth >= q as usize { - prop_assert!(result.is_ok(), "expected pass: {result:?}"); + prop_assert!(result.is_accepted(), "expected Accepted: {result:?}"); } else { - prop_assert!(result.is_err(), "expected too-shallow err: {result:?}"); + // i == j → strictness; i < j → candidate height below + // ancestor height. Both are proposer-controlled inputs, + // so they reject (vote nil) rather than error. + prop_assert!(result.is_rejected(), "expected Rejected: {result:?}"); } } } diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 320f885a9c8..ac82cf6ec04 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -20,7 +20,6 @@ use crate::{ MalachiteEvent, Mempool, ValidatorEntry, externalities::EthexeExternalities, mempool::TxInsertionStatus, types::ChainHead, }; -use alloy::providers::BoxedFut; use anyhow::Result; use ethexe_common::{ Address, SimpleBlockData, @@ -28,11 +27,11 @@ use ethexe_common::{ injected::SignedInjectedTransaction, }; use ethexe_malachite_core::MalachiteCore; -use futures::{FutureExt, Stream, stream::FusedStream}; +use futures::{Stream, stream::FusedStream}; use gprimitives::H256; use gsigner::schemes::secp256k1::PublicKey; use std::{ - collections::{HashMap, VecDeque}, + collections::HashMap, pin::Pin, sync::Arc, task::{Context, Poll}, diff --git a/ethexe/malachite/service/src/types.rs b/ethexe/malachite/service/src/types.rs index 469b11855fd..7405a7ada49 100644 --- a/ethexe/malachite/service/src/types.rs +++ b/ethexe/malachite/service/src/types.rs @@ -1,5 +1,4 @@ -use ethexe_common::{SimpleBlockData, injected::SignedInjectedTransaction}; -use gprimitives::H256; +use ethexe_common::SimpleBlockData; use tokio::sync::{Notify, RwLock}; pub struct ChainHead { diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index 47bf2ee5878..cec2f9a2a12 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -16,8 +16,6 @@ //! `globals.latest_finalized_mb_hash` is gap-free across the //! restart boundary, and the latest pointer never rewinds. -#![cfg(feature = "disable-tests")] - use std::{path::Path, time::Duration}; use async_trait::async_trait; @@ -43,11 +41,11 @@ struct EmptyMempool; #[async_trait] impl Mempool for EmptyMempool { - fn insert(&self, _tx: SignedInjectedTransaction) -> TxInsertionStatus { + async fn insert(&self, _tx: SignedInjectedTransaction) -> TxInsertionStatus { TxInsertionStatus::Inserted } - fn set_chain_head(&self, _head: SimpleBlockData) -> Vec { + async fn set_chain_head(&self, _head: SimpleBlockData) -> Vec { Vec::new() } From 9b31037c3aff1b34de0f98ec6d39c7ba11a588b8 Mon Sep 17 00:00:00 2001 From: Grisha Sobol Date: Thu, 11 Jun 2026 14:04:45 +0200 Subject: [PATCH 34/72] chore(vara.eth/malachite): add missing SPDX license headers Co-Authored-By: Claude Opus 4.7 --- ethexe/malachite/service/src/starter.rs | 3 +++ ethexe/malachite/service/src/types.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ethexe/malachite/service/src/starter.rs b/ethexe/malachite/service/src/starter.rs index 635529e2546..23c0d753c05 100644 --- a/ethexe/malachite/service/src/starter.rs +++ b/ethexe/malachite/service/src/starter.rs @@ -1,3 +1,6 @@ +// Copyright (C) Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + use crate::{ MalachiteEvent, MalachiteService, MalachiteServiceConfig, Mempool, config::ValidatorConfig, diff --git a/ethexe/malachite/service/src/types.rs b/ethexe/malachite/service/src/types.rs index 7405a7ada49..28ddfee0b9d 100644 --- a/ethexe/malachite/service/src/types.rs +++ b/ethexe/malachite/service/src/types.rs @@ -1,3 +1,6 @@ +// Copyright (C) Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + use ethexe_common::SimpleBlockData; use tokio::sync::{Notify, RwLock}; From a4e522316fc3638ffdddbd46e0e3879b1c6faad4 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 15:06:03 +0300 Subject: [PATCH 35/72] Simplify activation of replay boundary --- ethexe/service/src/fast_sync.rs | 11 ++++++----- ethexe/service/src/lib.rs | 7 ++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index a5091296273..f8a4c6b0432 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -586,7 +586,7 @@ async fn latest_era_with_committed_validators( .context("failed to calculate era from validators timestamp") } -pub(crate) async fn sync(service: &mut Service) -> Result { +pub(crate) async fn sync(service: &mut Service) -> Result<()> { let Service { observer, compute, @@ -599,7 +599,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result { } = service; let Some(network) = network else { log::warn!("Network service is disabled. Skipping fast synchronization..."); - return Ok(false); + return Ok(()); }; log::info!("Fast synchronization is in progress..."); @@ -637,7 +637,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result { EventData::collect(&block_loader, db, latest_block).await? else { log::warn!("No committed MB found. Skipping fast synchronization..."); - return Ok(false); + return Ok(()); }; log::warn!("No finalized committed MB found; trying latest block candidates"); state_query_block = latest_block; @@ -645,7 +645,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result { } None => { log::warn!("No committed MB found. Skipping fast synchronization..."); - return Ok(false); + return Ok(()); } }; @@ -730,6 +730,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result { }); if let Some(malachite) = malachite.as_mut() { + malachite.enable_replay_boundary_from_execution_snapshot(); malachite.receive_new_chain_head(block_data.to_simple()); } @@ -746,5 +747,5 @@ pub(crate) async fn sync(service: &mut Service) -> Result { }) .await; - Ok(true) + Ok(()) } diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 8d5668d1a43..f920d4e030b 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -578,11 +578,8 @@ impl Service { } pub async fn run(mut self) -> Result<()> { - if self.fast_sync - && fast_sync::sync(&mut self).await? - && let Some(malachite) = &self.malachite - { - malachite.enable_replay_boundary_from_execution_snapshot(); + if self.fast_sync { + fast_sync::sync(&mut self).await?; } self.run_inner().await.inspect_err(|err| { From b368b3bd0f813a55a2c983f192926d3b8c84f9b5 Mon Sep 17 00:00:00 2001 From: Grisha Sobol Date: Thu, 11 Jun 2026 14:32:45 +0200 Subject: [PATCH 36/72] docs(vara.eth/malachite): fix intra-doc links broken by type renames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `MalachiteConfig` → `MalachiteCoreConfig` / `MalachiteServiceConfig`, `MalachiteService` → `MalachiteCore` (in core), `MalachiteService::new` → `MalachiteServiceStarter::new`. CI docs build runs with `-D warnings`, so the stale links failed `build / docs`. Co-Authored-By: Claude Opus 4.7 --- ethexe/malachite/core/src/app.rs | 2 +- ethexe/malachite/core/src/config.rs | 6 +++--- ethexe/malachite/core/src/externalities.rs | 2 +- ethexe/malachite/core/src/lib.rs | 24 +++++++++++----------- ethexe/malachite/core/src/service.rs | 12 +++++------ ethexe/malachite/core/src/signing.rs | 2 +- ethexe/malachite/core/src/state.rs | 2 +- ethexe/malachite/core/src/types.rs | 2 +- ethexe/malachite/service/src/config.rs | 6 +++--- ethexe/malachite/service/src/lib.rs | 2 +- ethexe/service/src/config.rs | 4 ++-- ethexe/service/src/lib.rs | 2 +- 12 files changed, 33 insertions(+), 33 deletions(-) diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index 47b1f73de0e..7a98f41aa31 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -18,7 +18,7 @@ //! [`crate::Externalities`] is enforced by //! [`Store::cascade_save`] / [`Store::cascade_finalize`]; fatal //! callback errors are surfaced through -//! [`crate::MalachiteService`]'s error stream. +//! [`crate::MalachiteCore`]'s error stream. //! //! Each [`AppMsg`] variant is paired with a `process_*` method on //! [`AppMsgHandler`] that performs the work and returns the value the diff --git a/ethexe/malachite/core/src/config.rs b/ethexe/malachite/core/src/config.rs index 58ba5f8d1b0..d5a24d779c2 100644 --- a/ethexe/malachite/core/src/config.rs +++ b/ethexe/malachite/core/src/config.rs @@ -9,7 +9,7 @@ pub use malachitebft_app_channel::app::net::Multiaddr; /// One entry of the validator set. The set is fixed for the lifetime /// of the deployment — to rotate validators every node must be -/// re-bootstrapped from a fresh [`MalachiteConfig`]. +/// re-bootstrapped from a fresh [`MalachiteCoreConfig`]. // // TODO: #5480 add `libp2p_peer_id: PeerId` so receivers can gate // `ReceivedProposalPart` against a validator-peer-id allowlist @@ -36,11 +36,11 @@ pub struct ValidatorEntry { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum NodeRole { /// Sign votes and proposals; broadcast a validator proof on - /// connect; the local address must appear in [`MalachiteConfig::validators`]. + /// connect; the local address must appear in [`MalachiteCoreConfig::validators`]. Validator, /// Read-only participant — joins gossip / sync, validates /// incoming blocks, but never signs anything. The local address - /// must NOT appear in [`MalachiteConfig::validators`]. + /// must NOT appear in [`MalachiteCoreConfig::validators`]. FullNode, } diff --git a/ethexe/malachite/core/src/externalities.rs b/ethexe/malachite/core/src/externalities.rs index 05b8f0c448d..dda910194fc 100644 --- a/ethexe/malachite/core/src/externalities.rs +++ b/ethexe/malachite/core/src/externalities.rs @@ -64,7 +64,7 @@ pub trait Externalities: Send + Sync + 'static { /// The future may take an arbitrarily long time — for example to /// wait on a mempool, an external block source, or a chain head /// — and the service races it against - /// [`crate::MalachiteConfig::propose_timeout`]. On timeout the + /// [`crate::MalachiteCoreConfig::propose_timeout`]. On timeout the /// future is cancelled (dropped); implementations must be /// cancellation-safe. /// diff --git a/ethexe/malachite/core/src/lib.rs b/ethexe/malachite/core/src/lib.rs index 2b4093e0c7f..4b0c9c15dbd 100644 --- a/ethexe/malachite/core/src/lib.rs +++ b/ethexe/malachite/core/src/lib.rs @@ -9,7 +9,7 @@ //! can plug in without touching BFT plumbing. //! //! `ethexe-malachite` is the only direct consumer: it supplies the [`Externalities`] -//! implementation and wraps [`MalachiteService`] in its own facade exposed to the rest +//! implementation and wraps [`MalachiteCore`] in its own facade exposed to the rest //! of the ethexe stack. Block delivery happens exclusively through async //! [`Externalities`] callbacks. //! @@ -27,17 +27,17 @@ //! //! ## Usage //! -//! Construct the service with [`MalachiteService::new`](MalachiteService), then poll it +//! Construct the service with [`MalachiteCore::new`](MalachiteCore), then poll it //! as a `Stream`: //! //! ```rust,no_run -//! # use ethexe_malachite_core::{MalachiteConfig, MalachiteService, Externalities}; +//! # use ethexe_malachite_core::{MalachiteCoreConfig, MalachiteCore, Externalities}; //! # use std::sync::Arc; //! async fn start( -//! config: MalachiteConfig, +//! config: MalachiteCoreConfig, //! ext: Arc, -//! ) -> MalachiteService { -//! MalachiteService::::new(config, ext) +//! ) -> MalachiteCore { +//! MalachiteCore::::new(config, ext) //! .await //! .expect("service starts") //! } @@ -47,13 +47,13 @@ //! //! - [`Externalities`] — Async application callbacks: `process_mb_proposal`, `process_mb_finalized`, `build_block_above`, //! `validate_block_above`. -//! - [`MalachiteService`] — Running service; owns the swarm and store. Implements `Stream` and +//! - [`MalachiteCore`] — Running service; owns the swarm and store. Implements `Stream` and //! [`MService`]. `update_validators` rotates the active validator set, taking effect at the next height boundary. -//! - [`MService`] — Supertrait bound implemented by [`MalachiteService`]. +//! - [`MService`] — Supertrait bound implemented by [`MalachiteCore`]. //! - [`Block`] — Service-level block envelope: `{ parent_hash: H256, height: u64, payload: BlockPayload, reserved: [u8; 64] }`. -//! - [`ValidatorEntry`] — Validator set member: `public_key` + `voting_power`, used in [`MalachiteConfig`] and +//! - [`ValidatorEntry`] — Validator set member: `public_key` + `voting_power`, used in [`MalachiteCoreConfig`] and //! `update_validators`. -//! - [`MalachiteConfig`] — Node configuration: validator secret, validator set, `persistent_peers`, propose timeout, +//! - [`MalachiteCoreConfig`] — Node configuration: validator secret, validator set, `persistent_peers`, propose timeout, //! [`NodeRole`], `listen_addr`, and `base` project directory. //! - [`CommitCertificate`] — Finalization certificate delivered with `process_mb_finalized`. //! - [`libp2p_peer_id`] — Derive a [`PeerId`] from a validator secret offline, to build the `/p2p/` suffix of each @@ -61,14 +61,14 @@ //! //! ## Caller invariants //! -//! - `listen_addr` must be set explicitly; [`MalachiteConfig::DEFAULT_LISTEN_ADDR`] is +//! - `listen_addr` must be set explicitly; [`MalachiteCoreConfig::DEFAULT_LISTEN_ADDR`] is //! available but is not applied by any struct-level default. //! - `base` must be a persistent path: the service writes `/malachite/` (store and //! WAL) on first run and resumes from it on restart; a transient path loses BFT state. //! - Every persistent-peer multiaddr must include a `/p2p/` suffix (see //! [`libp2p_peer_id`]). //! - Returning `Err` from a proposal or finalization callback is fatal: the error -//! surfaces on the [`MalachiteService`] stream and the consensus loop aborts rather +//! surfaces on the [`MalachiteCore`] stream and the consensus loop aborts rather //! than skipping the callback. //! //! The service guarantees a strictly linearised block stream: each block is delivered diff --git a/ethexe/malachite/core/src/service.rs b/ethexe/malachite/core/src/service.rs index 0e7e5dc5253..32e2cc11d44 100644 --- a/ethexe/malachite/core/src/service.rs +++ b/ethexe/malachite/core/src/service.rs @@ -1,7 +1,7 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -//! [`MalachiteService`] — the public entry point. +//! [`MalachiteCore`] — the public entry point. use crate::{ app, @@ -80,7 +80,7 @@ const WAL_LOCK_POLL_INTERVAL: Duration = Duration::from_millis(25); /// The actor's `JoinHandle` is therefore *not* a sufficient barrier — /// the writer thread can still be live (and the lock still held) after /// the engine task exits. We probe the lock here so callers of -/// [`MalachiteService::shutdown`] can immediately re-open the same +/// [`MalachiteCore::shutdown`] can immediately re-open the same /// base dir without spurious "advisory lock held" errors. /// /// A missing WAL file means we never wrote one; nothing to wait for. @@ -180,7 +180,7 @@ impl MalachiteCore { // ---- validator set from config ---- if config.validators.is_empty() { - return Err(anyhow::anyhow!("MalachiteConfig::validators is empty")); + return Err(anyhow::anyhow!("MalachiteCoreConfig::validators is empty")); } let mut validators = Vec::with_capacity(config.validators.len()); for entry in &config.validators { @@ -197,7 +197,7 @@ impl MalachiteCore { NodeRole::Validator => { if !in_set { return Err(anyhow::anyhow!( - "NodeRole::Validator: local address {address} not present in MalachiteConfig::validators" + "NodeRole::Validator: local address {address} not present in MalachiteCoreConfig::validators" )); } let peer_id_bytes = libp2p_keypair.public().to_peer_id().to_bytes(); @@ -223,7 +223,7 @@ impl MalachiteCore { NodeRole::FullNode => { if in_set { return Err(anyhow::anyhow!( - "NodeRole::FullNode: local address {address} must NOT be in MalachiteConfig::validators" + "NodeRole::FullNode: local address {address} must NOT be in MalachiteCoreConfig::validators" )); } NetworkIdentity::new(moniker.clone(), libp2p_keypair, None) @@ -290,7 +290,7 @@ impl MalachiteCore { pub fn update_validators(&self, validators: Vec) -> Result<()> { if validators.is_empty() { return Err(anyhow::anyhow!( - "MalachiteService::update_validators: empty validators list" + "MalachiteCore::update_validators: empty validators list" )); } let mut converted = Vec::with_capacity(validators.len()); diff --git a/ethexe/malachite/core/src/signing.rs b/ethexe/malachite/core/src/signing.rs index a52f8d64f98..d4e4e588d19 100644 --- a/ethexe/malachite/core/src/signing.rs +++ b/ethexe/malachite/core/src/signing.rs @@ -5,7 +5,7 @@ //! derivation that the Malachite swarm uses. //! //! The node's master secret enters the service via -//! [`crate::MalachiteConfig::validator_secret`] (a +//! [`crate::MalachiteCoreConfig::validator_secret`] (a //! `gsigner::secp256k1::PrivateKey`). The 32 raw bytes drive two //! separate identities: //! diff --git a/ethexe/malachite/core/src/state.rs b/ethexe/malachite/core/src/state.rs index 83397e87b7a..6ac2e4f0b89 100644 --- a/ethexe/malachite/core/src/state.rs +++ b/ethexe/malachite/core/src/state.rs @@ -111,7 +111,7 @@ impl State { } /// Round timeouts. Propose phase is bounded by the configured - /// [`crate::MalachiteConfig::propose_timeout`] plus a small margin + /// [`crate::MalachiteCoreConfig::propose_timeout`] plus a small margin /// for non-proposers; everything else (including the per-round /// `propose_delta`) stays at the engine defaults. pub fn get_timeouts(&self, _height: Height) -> LinearTimeouts { diff --git a/ethexe/malachite/core/src/types.rs b/ethexe/malachite/core/src/types.rs index 5ff434f370b..801ec4bd952 100644 --- a/ethexe/malachite/core/src/types.rs +++ b/ethexe/malachite/core/src/types.rs @@ -1,7 +1,7 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -//! Core public types for [`crate::MalachiteService`]. +//! Core public types for [`crate::MalachiteCore`]. use gear_core::limited::LimitedVec; pub use gprimitives::H256; diff --git a/ethexe/malachite/service/src/config.rs b/ethexe/malachite/service/src/config.rs index 865fce8a845..c380759afef 100644 --- a/ethexe/malachite/service/src/config.rs +++ b/ethexe/malachite/service/src/config.rs @@ -37,7 +37,7 @@ pub struct MalachiteServiceConfig { /// Local libp2p listen address for the Malachite swarm. pub listen_addr: SocketAddr, - /// Directory where the wrapped [`ethexe_malachite_core::MalachiteService`] keeps + /// Directory where the wrapped [`ethexe_malachite_core::MalachiteCore`] keeps /// its WAL (`malachite/consensus.wal`) and RocksDB store /// (`malachite/store.db/`). pub home_dir: PathBuf, @@ -51,7 +51,7 @@ pub struct MalachiteServiceConfig { /// The complete validator set. The local node's public key (the /// one whose secret comes from the [`gsigner::Signer`] passed to - /// [`crate::MalachiteService::new`]) must appear in this list, or + /// [`crate::MalachiteServiceStarter::new`]) must appear in this list, or /// service start-up fails. /// /// Voting power is taken at face value — Tendermint's quorum @@ -78,7 +78,7 @@ impl MalachiteServiceConfig { /// Build a config with sane defaults from the node's home /// directory. The validator set is left empty — the caller MUST - /// fill it in before passing to [`crate::MalachiteService::new`] + /// fill it in before passing to [`crate::MalachiteServiceStarter::new`] /// (see [`Self::with_validators`]). pub fn from_home_dir(home_dir: PathBuf) -> Self { Self { diff --git a/ethexe/malachite/service/src/lib.rs b/ethexe/malachite/service/src/lib.rs index 7090bbd003f..b4e1f41a1ea 100644 --- a/ethexe/malachite/service/src/lib.rs +++ b/ethexe/malachite/service/src/lib.rs @@ -19,7 +19,7 @@ //! - [`MalachiteService`] (struct) — Public facade; `Stream` + driver methods //! - [`MalachiteEvent`] (enum) — Output event: proposal, finalization, purged txs //! - [`CommitCertificate`] (struct) — BFT commit proof attached to `BlockFinalized` -//! - [`MalachiteConfig`] (struct) — Service configuration +//! - [`MalachiteServiceConfig`] (struct) — Service configuration //! - [`ValidatorEntry`] (struct) — Single entry in the validator set //! - [`Mempool`] (trait) — Producer-side injected-tx source //! - [`InjectedTxMempool`] (struct) — Real mempool implementation diff --git a/ethexe/service/src/config.rs b/ethexe/service/src/config.rs index bf89e33a5bf..c27058bd607 100644 --- a/ethexe/service/src/config.rs +++ b/ethexe/service/src/config.rs @@ -21,7 +21,7 @@ pub struct Config { pub prometheus: Option, } -/// User-facing subset of [`ethexe_malachite::MalachiteConfig`], +/// User-facing subset of [`ethexe_malachite::MalachiteServiceConfig`], /// resolved at CLI/TOML parse time. The rest of the runtime fields /// (home directory, mempool) are filled in by the service itself. #[derive(Clone, Debug)] @@ -85,7 +85,7 @@ pub struct NodeConfig { pub canonical_quarantine: u8, /// Extra anchor-depth slack the proposer adds on top of /// `canonical_quarantine`. See - /// [`ethexe_malachite::MalachiteConfig::post_quarantine_delay`]. + /// [`ethexe_malachite::MalachiteServiceConfig::post_quarantine_delay`]. pub post_quarantine_delay: u32, pub dev: bool, pub pre_funded_accounts: u32, diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 5ecf8a427d4..22609592385 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -614,7 +614,7 @@ impl Service { /// Install a graceful-shutdown channel. The returned sender, /// when fired, breaks the run loop at the next yield and then - /// awaits [`MalachiteService::shutdown`] before `run` returns — + /// awaits [`ethexe_malachite::MalachiteService::shutdown`] before `run` returns — /// freeing the RocksDB advisory lock and libp2p listener /// synchronously, which a plain `JoinHandle::abort` does not /// guarantee. From 4385d9c48dcfc46ffd59c7e1c4c457bb7a2c1ba5 Mon Sep 17 00:00:00 2001 From: Gregory Sobol Date: Thu, 11 Jun 2026 14:50:31 +0200 Subject: [PATCH 37/72] docs example --- ethexe/malachite/service/src/externalities.rs | 79 ++++++------------- 1 file changed, 23 insertions(+), 56 deletions(-) diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index 8ddb65fb315..171dd585057 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -68,18 +68,28 @@ pub struct ExternalitiesConfig { } pub(crate) struct EthexeExternalities { + /// Shared DB reference for all storage operations pub db: Database, + /// Constant externalities config parameters pub cfg: ExternalitiesConfig, + /// Optional mempool reference for injected-tx processing; `None` when not a validator. pub mempool: Option>, + /// Reference to the latest chain head data. pub chain_head: Arc, + /// Pending service events queue. + /// Release events from here only when their prerequisite EB is prepared. pub pending_events: RwLock>, + /// Channel to poll events in MalachiteService. pub event_tx: UnboundedSender>, } /// One outbound [`MalachiteEvent`] that can't be released until its -/// `prerequisite` Eth block is fully synced into the local DB. +/// `prerequisite` Eth block is prepared in local DB. pub(crate) struct PendingEvent { + /// Event body pub event: MalachiteEvent, + /// Prerequisite Eth block hash + /// that must be prepared before this event can be emitted pub prerequisite: H256, } @@ -414,37 +424,13 @@ impl Externalities for EthexeExternalities { } } - // (4) Touched-programs cap (master's #6). Only enforced on - // the validator side — the proposer in `build_block_above` - // already shapes the MB to stay within the cap; this check - // is the participant's guard against a malicious proposer. - // - // Per master: `limit = max(initial_touched.len(), MAX_*)` — - // the proposer can't *avoid* programs already touched by EB - // events, so those set the floor for the cap. We add every - // `Operation::Injected` destination on top of the EB-touched - // seed and reject if the union exceeds `limit`. - // - // NOTE: there is no per-MB size cap on the validator side - // (master parity). We rely on the Malachite engine's 1 MiB - // hard cap on the encoded `Block` payload — anything larger - // never reaches `validate_block_above` in the first place. - let parent_advanced = if parent_hash.is_zero() { - H256::zero() - } else { - self.db.mb_meta(parent_hash).last_advanced_eb - }; - // `?` here only fires on local DB issues: missing - // `mb_program_states` for `latest_computed_mb_hash`, missing - // `block_header` on a canonical ancestor of `advance`, or - // missing `block_events` for one of them. After the quarantine - // gate above succeeded the observer has clearly synced - // `advance` and its ancestors, so any failure here is a local - // DB / sync race — not a proposer-controlled condition. Same - // reasoning as the other two `?`s in this function. + let parent_advanced = parent_hash + .is_zero() + .then(H256::zero) + .unwrap_or_else(|| self.db.mb_meta(parent_hash).last_advanced_eb); let mut touched = match advance { Some(advanced_eb) => eb_touched_programs(&self.db, parent_advanced, advanced_eb)?, - None => std::collections::HashSet::new(), + None => Default::default(), }; let limit = touched.len().max(MAX_TOUCHED_PROGRAMS_PER_MB as usize); for tx in payload.iter() { @@ -464,31 +450,14 @@ impl Externalities for EthexeExternalities { } impl EthexeExternalities { - /// True iff `prerequisite.is_zero()` (no prerequisite — genesis - /// or pre-advance) or the prerequisite Eth block has been fully - /// **prepared** locally. - /// - /// "Prepared" (vs. merely observed via `block_events`) is the - /// stronger condition we need: `prepare_block`'s pipeline - /// transitions through `WaitingForCodes` and only flips - /// `block_meta.prepared = true` once every code referenced by - /// the block (and its ancestors) has been loaded and validated. - /// Releasing the BlockProposal event on merely-observed (but not - /// yet prepared) EBs would let downstream `compute_mb` race the - /// code-validation pipeline and fail with `MissingCode` when an - /// MB's advance chain contains a `ProgramCreated` event for a - /// not-yet-validated code. + /// Check whether the prerequisite EB is prepared in local DB. + /// Zero hash is a special case that always passes. fn prerequisite_satisfied(&self, prerequisite: H256) -> bool { use ethexe_common::db::BlockMetaStorageRO; prerequisite.is_zero() || self.db.block_meta(prerequisite).prepared } - /// Forward `event` to the outbound channel right away when its - /// `prerequisite` Eth block is locally synced AND no earlier - /// queued event is still waiting; otherwise push it onto the - /// pending buffer to keep ordering. Held entries are released - /// from the front by [`Self::drain_pending_events`] once their - /// prerequisite lands. + /// Send event immediately if prerequisite is satisfied, otherwise queue it for later emission. pub(crate) async fn try_emit_or_queue(&self, event: MalachiteEvent, prerequisite: H256) { let mut queue = self.pending_events.write().await; if queue.is_empty() && self.prerequisite_satisfied(prerequisite) { @@ -502,11 +471,7 @@ impl EthexeExternalities { } } - /// Pop and emit pending events from the front while their - /// prerequisite is satisfied. Stops at the first still-blocked - /// entry so ordering is preserved (later events may have a - /// later prerequisite, but FIFO drain only releases what's - /// safely ready right now). + /// Check the pending events queue and release any events whose prerequisites are now satisfied. pub(crate) async fn drain_pending_events(&self) { let mut queue = self.pending_events.write().await; while let Some(front) = queue.front() { @@ -551,7 +516,9 @@ impl EthexeExternalities { } } - // Candidate EB must be anchored in the quarantine and a strict descendant of the previously advanced EB. + // Find an EB candidate that can be advanced to according to the current chain head: + // 1. Should pass quarantine with post quarantine delay against the latest synced EB. + // 2. Should be a strict descendant of the previously advanced EB. async fn find_eb_candidate_for_advancing(&self, parent_advance: H256) -> Result> { let chain_head = *self.chain_head.latest_synced.read().await; let start = self.db.globals().start_block_hash; From f154c85e86536296d5707c46c80db4ae6dc656b4 Mon Sep 17 00:00:00 2001 From: Grisha Sobol Date: Thu, 11 Jun 2026 15:20:24 +0200 Subject: [PATCH 38/72] docs(vara.eth/malachite): concise item-level documentation Apply the short, informative doc style across ethexe-malachite and ethexe-malachite-core: one-line field docs, compressed method/struct docs without internal narration, trimmed inline comment blocks. Crate-level docs untouched. Co-Authored-By: Claude Opus 4.7 --- ethexe/malachite/core/src/app.rs | 35 +---- ethexe/malachite/core/src/config.rs | 79 +++------- ethexe/malachite/core/src/externalities.rs | 100 ++++--------- ethexe/malachite/core/src/service.rs | 72 +++------ ethexe/malachite/core/src/signing.rs | 61 ++------ ethexe/malachite/core/src/state.rs | 41 +++-- ethexe/malachite/core/src/store.rs | 56 +++---- ethexe/malachite/core/src/types.rs | 50 ++----- ethexe/malachite/service/src/config.rs | 53 +++---- ethexe/malachite/service/src/externalities.rs | 8 +- ethexe/malachite/service/src/lib.rs | 5 +- ethexe/malachite/service/src/mempool.rs | 74 +++------ ethexe/malachite/service/src/service.rs | 31 ++-- ethexe/malachite/service/src/starter.rs | 7 +- ethexe/malachite/service/src/tx_validity.rs | 140 +++++------------- ethexe/malachite/service/src/types.rs | 5 + 16 files changed, 264 insertions(+), 553 deletions(-) diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index 7a98f41aa31..15abe12e631 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -634,22 +634,10 @@ where } } - /// Drive the strict-ordering save cascade against the application - /// for a freshly-assembled block. Inserts a `saved=false, - /// finalized=false, cert=None` [`BlockEntry`] and runs - /// [`Store::cascade_save`] from this hash — the cascade flushes - /// every ancestor that is now connected in chronological - /// (parent-first) order. If an ancestor is still missing the - /// cascade is a no-op; it will pick up when the gap closes via a - /// later assembled block at the missing height. - /// - /// Called from [`Self::process_get_value`] (we are proposer), - /// [`Self::process_received_proposal_part`] (peer proposal), and - /// [`Self::process_synced_value`] (sync path). Multiple callers - /// can race for the same `block_hash` — `Store::insert_block` - /// dedup is idempotent and `cascade_save` skips already-saved - /// entries, so the application's `process_mb_proposal` runs at - /// most once per `block_hash`. + /// Record a freshly-assembled block and run the parent-first save + /// cascade, so the application's `process_mb_proposal` fires for every + /// now-connected ancestor. Idempotent — racing callers are deduped and + /// the callback runs at most once per `block_hash`. async fn record_assembled_block(&self, block: Block) -> Result<()> { let block_hash = block.hash(); self.state.store.insert_block(BlockEntry { @@ -671,17 +659,10 @@ where .await } - /// Attach the engine's quorum certificate to the - /// already-processed [`BlockEntry`] and run the finalize cascade. - /// - /// Contract: the block (and every ancestor) must have already been - /// processed by [`Self::record_assembled_block`] earlier — the - /// strict-ordering guarantee documented on - /// [`crate::Externalities::process_mb_proposal`]. A debug-build - /// assertion catches a violation; in release builds - /// [`Store::cascade_finalize`] silently no-ops on an unsaved - /// ancestor (the `finalize_chain` walk returns `None`), and the - /// `errors_tx` channel surfaces the contract breach upstream. + /// Attach the engine's quorum certificate to the already-processed + /// [`BlockEntry`] and run the finalize cascade. The block and every + /// ancestor must have been saved via [`Self::record_assembled_block`] + /// first (debug assertion; release no-ops on an unsaved ancestor). async fn ingest_finalized(&self, cert: EngineCert, block_bytes: Vec) -> Result<()> { let block = Block::decode(&mut &block_bytes[..]) .map_err(|e| anyhow!("decoding Block at finalize: {e}"))?; diff --git a/ethexe/malachite/core/src/config.rs b/ethexe/malachite/core/src/config.rs index d5a24d779c2..d793bba6658 100644 --- a/ethexe/malachite/core/src/config.rs +++ b/ethexe/malachite/core/src/config.rs @@ -7,9 +7,7 @@ use std::{net::SocketAddr, path::PathBuf, time::Duration}; pub use malachitebft_app_channel::app::net::Multiaddr; -/// One entry of the validator set. The set is fixed for the lifetime -/// of the deployment — to rotate validators every node must be -/// re-bootstrapped from a fresh [`MalachiteCoreConfig`]. +/// One entry of the validator set. // // TODO: #5480 add `libp2p_peer_id: PeerId` so receivers can gate // `ReceivedProposalPart` against a validator-peer-id allowlist @@ -26,91 +24,54 @@ pub struct ValidatorEntry { } /// Role this node plays in the BFT swarm. -/// -/// A `FullNode` doesn't propose or vote — it joins the gossip mesh, -/// receives proposals + sync responses, and surfaces them to the -/// application via [`crate::Externalities::process_mb_proposal`] / -/// [`crate::Externalities::process_mb_finalized`] just like a -/// validator would. Use this for read-only observers, -/// quarantine workers, light clients, etc. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum NodeRole { - /// Sign votes and proposals; broadcast a validator proof on - /// connect; the local address must appear in [`MalachiteCoreConfig::validators`]. + /// Signs votes and proposals; the local key must appear in + /// [`MalachiteCoreConfig::validators`]. Validator, - /// Read-only participant — joins gossip / sync, validates - /// incoming blocks, but never signs anything. The local address - /// must NOT appear in [`MalachiteCoreConfig::validators`]. + /// Read-only participant — joins gossip / sync and validates blocks, + /// but never signs; the local key must NOT be in the validator set. FullNode, } -/// All configuration the service needs to bootstrap the malachite -/// engine. -/// -/// Application-specific knobs (gas budgets, mempool settings, etc.) -/// live behind [`crate::Externalities`] — they don't belong here. +/// All configuration the service needs to bootstrap the malachite engine. +/// Application-specific knobs live behind [`crate::Externalities`]. #[derive(Clone, Debug)] pub struct MalachiteCoreConfig { /// Local libp2p listen address. pub listen_addr: SocketAddr, - /// Application's project base directory. The service carves out - /// `/malachite/` and owns everything inside it: the - /// consensus WAL (`consensus.wal`) and the RocksDB store - /// (`store.db/` — block entries, decided/undecided proposals, - /// pending parts, height index, engine certificates). Anything - /// else under `base` is the application's business. - /// - /// The artifacts inside `/malachite/` are created on first - /// run; subsequent runs resume from where the previous one left - /// off. - /// - /// In tests, the caller is responsible for keeping this directory - /// alive across service restarts (don't drop the `TempDir` between - /// service spawns). + /// Base directory; the service owns `/malachite/` (consensus WAL + /// + RocksDB store), created on first run and resumed on restarts. pub base: PathBuf, - /// Multiaddrs the local node should keep persistent connections - /// to. Each entry must include the `/p2p/` suffix so the - /// swarm knows who to expect on the other side. Discovery is off, - /// so multi-validator deployments need every node's multiaddr - /// listed (or at least transitively reachable). + /// Multiaddrs of peers to keep persistent connections to; each entry + /// must include a `/p2p/` suffix (discovery is off). pub persistent_peers: Vec, - /// This node's secp256k1 secret. Used (after a domain-separated - /// derivation) for the libp2p peer identity in both roles, and - /// additionally for malachite vote / proposal signing in - /// [`NodeRole::Validator`] mode. + /// This node's secp256k1 secret: libp2p peer identity in both roles, + /// plus vote / proposal signing in [`NodeRole::Validator`] mode. pub validator_secret: gsigner::schemes::secp256k1::PrivateKey, - /// Validator set the engine uses to drive consensus. For - /// [`NodeRole::Validator`] the set must contain an entry whose - /// public key matches [`Self::validator_secret`]; for - /// [`NodeRole::FullNode`] the local key must NOT be in the set. + /// Validator set the engine uses to drive consensus + /// (see [`NodeRole`] for local-key membership rules). pub validators: Vec, /// Whether this node casts votes (`Validator`) or just observes /// (`FullNode`). pub role: NodeRole, - /// Upper bound on how long the service will wait on - /// [`crate::Externalities::build_block_above`] before giving up - /// and letting malachite's round timeout advance the proposer. + /// Upper bound on waiting for [`crate::Externalities::build_block_above`] + /// before the round rolls over. pub propose_timeout: Duration, } impl MalachiteCoreConfig { - /// Default propose timeout — 13 seconds. The upper bound on how - /// long [`crate::Externalities::build_block_above`] is given to - /// produce a block before the round rolls over. Applications - /// should override this when they have a faster or slower - /// block-production deadline. + /// Default propose timeout. pub const DEFAULT_PROPOSE_TIMEOUT: Duration = Duration::from_secs(13); - /// Default libp2p listen address — `0.0.0.0:20334`. Sits next to - /// the typical 20333/udp QUIC port commonly used for - /// application-level networking, but on TCP since malachite's - /// default transport is TCP. + /// Default libp2p listen address — TCP next to the typical + /// 20333/udp application QUIC port. pub const DEFAULT_LISTEN_ADDR: SocketAddr = SocketAddr::new( std::net::IpAddr::V4(std::net::Ipv4Addr::new(0, 0, 0, 0)), 20334, diff --git a/ethexe/malachite/core/src/externalities.rs b/ethexe/malachite/core/src/externalities.rs index dda910194fc..ad52163c86c 100644 --- a/ethexe/malachite/core/src/externalities.rs +++ b/ethexe/malachite/core/src/externalities.rs @@ -9,93 +9,43 @@ use async_trait::async_trait; use ethexe_common::Acceptance; /// Application-side callbacks the consensus service requires. +/// The application contributes only the opaque [`BlockPayload`]; +/// everything BFT-related stays inside the service. /// -/// The service is application-agnostic: it owns the BFT engine, the -/// libp2p swarm, and the persistent BFT state. The opaque, size-capped -/// payload byte string ([`BlockPayload`]) is the only -/// shape the application contributes to a [`Block`] — encoding and -/// decoding of any application-level schema lives behind this trait. +/// Guaranteed happens-before ordering (`H256::zero()` parent = genesis): /// -/// The service guarantees a strict happens-before ordering for the -/// callbacks below — the application never has to maintain its own -/// synchronization barrier: -/// -/// 1. [`Self::process_mb_proposal`] for `mb_hash` is called as soon -/// as a proposal carrying that hash has been assembled and -/// validated locally (regardless of whether this node is the -/// proposer, a participant that received the proposal over the -/// network, or a full node that received a synced decided value), -/// but **only after** every ancestor of `mb_hash` has already -/// returned successfully from a previous `process_mb_proposal` -/// call. Sibling proposals at the same height are possible (one -/// per fork-causing round) and each is delivered exactly once -/// per `mb_hash`. -/// 2. [`Self::process_mb_finalized`] for `mb_hash` is called **only -/// after** `process_mb_proposal` for that same `mb_hash` returned -/// successfully **and** every ancestor has already been finalized -/// via previous `process_mb_finalized` calls. -/// 3. [`Self::build_block_above`] / [`Self::validate_block_above`] -/// are called only after the parent has been finalized (or -/// `parent_hash == H256::zero()` when building / validating the -/// genesis block). -/// -/// All methods are async; the service `await`s them inline. +/// 1. [`Self::process_mb_proposal`] fires once per `mb_hash`, only after +/// every ancestor's `process_mb_proposal` returned `Ok`. Sibling +/// proposals at the same height are possible. +/// 2. [`Self::process_mb_finalized`] fires only after the same hash's +/// `process_mb_proposal` and every ancestor's finalization. +/// 3. [`Self::build_block_above`] / [`Self::validate_block_above`] fire +/// only after the parent has been finalized. #[async_trait] pub trait Externalities: Send + Sync + 'static { - /// Persist `block` indexed by `mb_hash`. Called exactly once - /// per `mb_hash` over the lifetime of an application instance, - /// at proposal-assembly time, after every ancestor's - /// `process_mb_proposal` has already returned `Ok`. + /// Persist `block` indexed by `mb_hash`; called exactly once per hash + /// at proposal-assembly time. async fn process_mb_proposal(&self, mb_hash: H256, block: Block) -> Result<()>; - /// Mark `mb_hash` as finalized and durable. - /// - /// `cert` is the BFT commit certificate for the height of - /// `mb_hash`. The application typically forwards `cert` to - /// downstream layers (on-chain commits, light clients, etc.). + /// Mark `mb_hash` as finalized; `cert` is the BFT commit certificate + /// for its height. async fn process_mb_finalized(&self, mb_hash: H256, cert: CommitCertificate) -> Result<()>; - /// Build a fresh block payload whose parent has hash - /// `parent_mb_hash`. Called only when this node has been elected - /// proposer. The new block's height is derivable from `parent_mb_hash` - /// (parent.height + 1, or 1 for genesis), so it isn't passed - /// explicitly here. - /// - /// The future may take an arbitrarily long time — for example to - /// wait on a mempool, an external block source, or a chain head - /// — and the service races it against - /// [`crate::MalachiteCoreConfig::propose_timeout`]. On timeout the - /// future is cancelled (dropped); implementations must be - /// cancellation-safe. + /// Build a fresh block payload on top of `parent_mb_hash`; called only + /// when this node is the elected proposer. /// - /// `parent_hash == H256::zero()` is passed when building the - /// genesis block. + /// The future may wait arbitrarily long — the service races it against + /// [`crate::MalachiteCoreConfig::propose_timeout`] and cancels it on + /// timeout, so implementations must be cancellation-safe. async fn build_block_above(&self, parent_mb_hash: H256) -> Result; - /// Application-side validation of an incoming proposal's - /// **payload only**. - /// - /// Parent linkage and height progression are validated inside - /// the consensus layer before this hook fires; the caller still - /// passes `parent_mb_hash` for context (e.g. to read ancestor state - /// from an application-side store) but is not expected to - /// re-check `block.parent_mb_hash`. `parent_mb_hash == H256::zero()` - /// signals the genesis block. - /// - /// Typical responsibilities: - /// - the payload bytes decode against the application's schema; - /// - the decoded content is well-formed against the application's - /// protocol invariants (gas budget, single anchor advance, - /// operation shape, etc.). - /// - Optionally a stronger proposer-authorization check on top - /// of malachite's validator set. - /// - /// Returns `Ok(true)` to vote for the proposal, `Ok(false)` to - /// reject without crashing, `Err(_)` for an unexpected internal - /// failure (surfaces as an error event on the service stream). + /// Application-side validation of an incoming proposal's **payload + /// only** — parent linkage and height are checked by the consensus + /// layer before this hook fires. /// - /// Not called on the sync path — sync values come with a quorum - /// commit certificate and are accepted on that basis alone. + /// Returns `Accepted` to vote for the proposal, `Rejected(reason)` to + /// vote nil, `Err(_)` for an unexpected internal failure. Not called + /// on the sync path (sync values carry a quorum certificate). async fn validate_block_above( &self, parent_mb_hash: H256, diff --git a/ethexe/malachite/core/src/service.rs b/ethexe/malachite/core/src/service.rs index 32e2cc11d44..7be2c95a1cc 100644 --- a/ethexe/malachite/core/src/service.rs +++ b/ethexe/malachite/core/src/service.rs @@ -49,41 +49,30 @@ pub trait MService: Stream + Send + Unpin {} /// Application-agnostic Malachite BFT consensus service. pub struct MalachiteCore { + /// Fatal errors forwarded from the app task. errors_rx: mpsc::UnboundedReceiver, + /// Handle to the malachite engine actor tree. engine: EngineHandle, + /// Handle to the spawned app event-loop task. app_handle: JoinHandle<()>, - /// Path to the WAL file. [`Self::shutdown`] probes the advisory - /// lock on this path before returning so the next service - /// opening the same base dir does not race the WAL writer thread. + /// WAL file path; [`Self::shutdown`] probes its advisory lock before + /// returning so a restart on the same base dir doesn't race the writer. wal_path: PathBuf, - /// Shared with the inner app loop; [`Self::update_validators`] - /// writes here, the next `Finalized` / `ConsensusReady` reply reads. + /// Shared with the app loop; [`Self::update_validators`] writes here. validator_set: SharedValidatorSet, + /// Keeps the externalities alive for the app task. _externalities: Arc, } -/// Upper bound on how long [`MalachiteCore::shutdown`] will wait -/// for the WAL advisory lock to be released after the engine actor -/// has stopped. Empirically the writer thread drops the file within -/// tens of milliseconds; this ceiling guards against pathological CI -/// scheduling without ever blocking healthy shutdowns. +/// Upper bound on how long [`MalachiteCore::shutdown`] waits for the WAL +/// advisory lock to release after the engine actor has stopped. const WAL_LOCK_RELEASE_TIMEOUT: Duration = Duration::from_secs(10); const WAL_LOCK_POLL_INTERVAL: Duration = Duration::from_millis(25); -/// Block until the WAL file at `wal_path` is no longer locked, or -/// `timeout` has elapsed. -/// -/// Malachite's WAL is owned by a [`std::thread`] (see -/// `arc-malachitebft-engine`'s `wal::thread`); the engine actor's -/// `post_stop` only sends a `Shutdown` message on a channel, and the -/// thread releases its [`advisory_lock`] when it later drops the log. -/// The actor's `JoinHandle` is therefore *not* a sufficient barrier — -/// the writer thread can still be live (and the lock still held) after -/// the engine task exits. We probe the lock here so callers of -/// [`MalachiteCore::shutdown`] can immediately re-open the same -/// base dir without spurious "advisory lock held" errors. -/// -/// A missing WAL file means we never wrote one; nothing to wait for. +/// Block until the WAL file is no longer locked or `timeout` elapses. +/// The WAL writer is a detached thread, so the engine actor's JoinHandle +/// is not a sufficient barrier; probing the lock lets the caller re-open +/// the same base dir right away. A missing WAL file passes immediately. async fn wait_wal_lock_released(wal_path: &Path, timeout: Duration) { let deadline = tokio::time::Instant::now() + timeout; loop { @@ -114,12 +103,8 @@ async fn wait_wal_lock_released(wal_path: &Path, timeout: Duration) { impl Drop for MalachiteCore { fn drop(&mut self) { - // Stop the engine actor so its libp2p / consensus children - // shut down cleanly, then abort the app and engine join handles. - // Note: this is a fire-and-forget shutdown — RocksDB locks - // and listening sockets may take a few hundred ms to release. - // Use [`Self::shutdown`] for tests that immediately re-open - // the same home directory. + // Fire-and-forget shutdown; locks and sockets may take a moment to + // release. Use [`Self::shutdown`] when re-opening the same base dir. self.engine.actor.kill(); self.app_handle.abort(); self.engine.handle.abort(); @@ -127,21 +112,15 @@ impl Drop for MalachiteCore { } impl MalachiteCore { - /// Block until the engine actor tree has finished shutting down - /// and any open file locks (RocksDB, WAL) have been released. - /// Use this before re-opening the same `base` to avoid - /// "advisory lock held" errors at the second `new()` call. + /// Block until the engine actor tree has shut down and the file locks + /// (RocksDB, WAL) are released — required before re-opening the same `base`. pub async fn shutdown(mut self) { self.engine.actor.kill(); - // `kill` is asynchronous — the actor finishes its current - // message and then stops, so we await the JoinHandles. + // `kill` is asynchronous — await the JoinHandles. let _ = (&mut self.engine.handle).await; self.app_handle.abort(); let _ = (&mut self.app_handle).await; - // The engine task exiting doesn't synchronously release the - // WAL advisory lock — the writer is a detached std::thread. - // Probe the lock so callers can immediately re-open the same - // base dir. + // The WAL writer is a detached thread; probe its lock explicitly. wait_wal_lock_released(&self.wal_path, WAL_LOCK_RELEASE_TIMEOUT).await; } } @@ -278,15 +257,10 @@ impl MalachiteCore { }) } - /// Swap the active validator set used at the next height start. - /// Malachite's `StartHeight` snapshots the set at the height - /// start, so the current height runs to completion with whatever - /// it had; the `Finalized` reply then feeds the new set as the - /// next-height `HeightParams`, keeping the rotation gap-free. - /// - /// Caller is responsible for keeping the local validator's pub - /// key in `validators` while running in [`NodeRole::Validator`] - /// — we don't carry the role around here. Empty input is rejected. + /// Swap the active validator set, taking effect at the next height start + /// (the current height runs to completion with the old set). + /// The caller must keep the local key in the set while in + /// [`NodeRole::Validator`]. Empty input is rejected. pub fn update_validators(&self, validators: Vec) -> Result<()> { if validators.is_empty() { return Err(anyhow::anyhow!( diff --git a/ethexe/malachite/core/src/signing.rs b/ethexe/malachite/core/src/signing.rs index d4e4e588d19..c790a182102 100644 --- a/ethexe/malachite/core/src/signing.rs +++ b/ethexe/malachite/core/src/signing.rs @@ -2,26 +2,11 @@ // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 //! secp256k1 / ECDSA signing primitives plus the libp2p identity -//! derivation that the Malachite swarm uses. +//! derivation for the Malachite swarm. //! -//! The node's master secret enters the service via -//! [`crate::MalachiteCoreConfig::validator_secret`] (a -//! `gsigner::secp256k1::PrivateKey`). The 32 raw bytes drive two -//! separate identities: -//! -//! - the consensus signer ([`MalachiteSigner`]) — signs Malachite -//! votes / proposals / `Fin` parts; -//! - a domain-separated libp2p keypair — independent peer-id so a -//! process running another libp2p swarm under the same key doesn't -//! collide. -//! -//! Address derivation is the standard -//! `keccak256(uncompressed_pubkey[1..])[12..]` flow. The 20-byte -//! address sits inside [`crate::Address`] as a gsigner newtype. -//! -//! The malachite-side `SigningProvider` impl for -//! [`MalachiteSigner`] lives in [`crate::context`] alongside the -//! `Context` type it parametrises. +//! [`crate::MalachiteCoreConfig::validator_secret`] drives two identities: +//! the consensus signer ([`MalachiteSigner`]) and a domain-separated +//! libp2p keypair. use anyhow::{Context as _, Result}; use libp2p_identity::{Keypair, PeerId}; @@ -38,10 +23,8 @@ pub type PublicKey = malachitebft_signing_ecdsa::PublicKey; /// Concrete ECDSA signature on the k256 curve. pub type Signature = malachitebft_signing_ecdsa::Signature; -/// Local signing helper, the consensus side of the validator -/// identity. Owns the private key for the lifetime of the service -/// and exposes the small set of operations the malachite layer -/// needs. +/// Consensus-side signer of the validator identity; owns the private key +/// for the lifetime of the service. #[derive(Debug)] pub struct MalachiteSigner { private_key: PrivateKey, @@ -79,8 +62,7 @@ impl MalachiteSigner { // Helpers // --------------------------------------------------------------------------- -/// Pack a [`Signature`] into a `Vec` (raw `r || s` for the -/// k256 curve, 64 bytes). Helper used by the SCALE codec layer. +/// Pack a [`Signature`] into raw `r || s` bytes (64 bytes). pub fn signature_to_vec(s: &Signature) -> Vec { s.to_vec() } @@ -90,28 +72,21 @@ pub fn signature_from_vec(bytes: &[u8]) -> Result { Signature::from_slice(bytes).map_err(|e| anyhow::anyhow!("decoding signature from bytes: {e}")) } -/// Construct an ECDSA private key from a raw 32-byte secret. Returns -/// an error if the bytes are not a valid k256 scalar (zero or ≥ curve -/// order); for randomly drawn secrets this is overwhelmingly unlikely -/// (≈ 2^-128) but real input may come from anywhere. +/// Construct an ECDSA private key from a raw 32-byte secret. +/// Errors if the bytes are not a valid k256 scalar. pub fn private_key_from_bytes(secret: &[u8; 32]) -> Result { PrivateKey::from_slice(secret) .map_err(|e| anyhow::anyhow!("constructing ECDSA private key: {e}")) } -/// Convert a `gsigner` secp256k1 [`PrivateKey`] into the malachite- -/// side [`PrivateKey`]. Both are k256-backed, so this is a -/// bytes-roundtrip. +/// Convert a `gsigner` secp256k1 private key into the malachite-side one. pub fn private_key_from_gsigner( pk: &gsigner::schemes::secp256k1::PrivateKey, ) -> Result { private_key_from_bytes(&pk.to_bytes()) } -/// Convert a `gsigner` secp256k1 [`PublicKey`] into the malachite- -/// side [`PublicKey`]. Both are k256-backed; gsigner stores the -/// 33-byte SEC1 compressed form, which malachite accepts via -/// `from_sec1_bytes`. +/// Convert a `gsigner` secp256k1 public key into the malachite-side one. pub fn public_key_from_gsigner(pk: &gsigner::schemes::secp256k1::PublicKey) -> Result { let bytes = pk.to_bytes(); PublicKey::from_sec1_bytes(&bytes) @@ -133,11 +108,8 @@ pub fn address_bytes_from_public_key(pk: &PublicKey) -> [u8; 20] { out } -/// Derive the libp2p secp256k1 secret used by the Malachite swarm -/// from the validator's master secret. Domain-separated so two -/// libp2p swarms under the same validator key (e.g. an application -/// network on QUIC plus the malachite TCP transport) don't collide -/// peer-ids. +/// Derive the Malachite swarm's libp2p secret from the validator's master +/// secret; domain-separated so swarms under the same key don't collide. pub fn derive_libp2p_secret(validator_secret: &[u8; 32]) -> [u8; 32] { const DOMAIN: &[u8] = b"mala-svc-libp2p:v1:"; let mut h = Keccak256::new(); @@ -159,11 +131,8 @@ pub fn libp2p_keypair_from(validator_secret: &[u8; 32]) -> Keypair { Keypair::from(inner) } -/// Compute the libp2p [`PeerId`] of the Malachite swarm associated -/// with `validator_secret` without spinning up the engine. Useful for -/// offline tooling: operators preparing `--persistent-peer` multiaddrs -/// can compute the `/p2p/` suffix from each validator's -/// keystore without booting the node. +/// Compute the Malachite swarm's libp2p [`PeerId`] offline — lets +/// operators build `/p2p/` suffixes without booting a node. pub fn libp2p_peer_id(validator_secret: &[u8; 32]) -> PeerId { libp2p_keypair_from(validator_secret).public().to_peer_id() } diff --git a/ethexe/malachite/core/src/state.rs b/ethexe/malachite/core/src/state.rs index 6ac2e4f0b89..efa0fb465bc 100644 --- a/ethexe/malachite/core/src/state.rs +++ b/ethexe/malachite/core/src/state.rs @@ -69,15 +69,25 @@ impl SharedValidatorSet { } } +/// Volatile bookkeeping of the app event loop. pub(crate) struct State { + /// Consensus signer of the local node. pub signer: MalachiteSigner, + /// Active validator set handle. pub validator_set: SharedValidatorSet, + /// Local node's address. pub address: Address, + /// Persistent block store. pub store: Store, + /// Per-peer proposal-part stream reassembly. streams_map: PartStreamsMap, + /// Height the engine is currently working on. pub current_height: Height, + /// Round within the current height. pub current_round: Round, + /// Proposer of the current round, once known. pub current_proposer: Option
, + /// Deadline for `build_block_above`. pub propose_timeout: Duration, } @@ -134,13 +144,8 @@ impl State { self.streams_map.insert(from, part) } - /// Re-assemble a [`ProposedValue`] from a completed - /// [`ProposalParts`] sequence. The single `Data` part carries - /// the SCALE-encoded block bytes; `Init` supplies the (height, - /// round, proposer) header. Validation is the caller's - /// responsibility — application-level checks happen via - /// [`crate::Externalities::validate_block_above`] and the - /// `ProposalFin` signature check (when wired in). + /// Re-assemble a [`ProposedValue`] from a completed [`ProposalParts`] + /// sequence. Validation is the caller's responsibility. pub fn assemble_value_from_parts(parts: ProposalParts) -> Result> { let init = parts.init().ok_or_else(|| anyhow!("missing Init part"))?; let block_bytes = parts @@ -163,9 +168,8 @@ impl State { // ----------------------- propose-side helpers --------------------- - /// Wrap a freshly-built block payload into a - /// [`LocallyProposedValue`] for the engine. The block is - /// SCALE-encoded once here and stays in that form on the wire. + /// Wrap a freshly-built block payload into a [`LocallyProposedValue`] + /// for the engine and persist it as an undecided proposal. pub fn build_locally_proposed_value( &mut self, height: Height, @@ -192,10 +196,8 @@ impl State { )) } - /// Reuse a prior locally-built value if the engine re-asks - /// `GetValue` for the same `(height, round)`. Avoids wasted - /// block-build work and prevents non-determinism (proposer might - /// otherwise build different content the second time). + /// Reuse a prior locally-built value when the engine re-asks `GetValue` + /// for the same `(height, round)` — avoids rebuild non-determinism. pub fn get_previously_built_value( &self, height: Height, @@ -235,14 +237,9 @@ impl State { }) } - /// Commit a finalized value: pull the matching undecided proposal - /// out of the engine store, persist the decided value + cert, and - /// advance to the next height. - /// - /// Returns the committed block bytes (SCALE-encoded - /// [`crate::Block`]) so the caller (`app.rs`) can decode it, - /// compute the [`crate::H256`] block hash, and insert into the - /// [`crate::store::BlockEntry`] layer. + /// Commit a finalized value: persist the decided value + cert and + /// advance to the next height. Returns the committed block bytes + /// (SCALE-encoded [`crate::Block`]). pub fn commit( &mut self, certificate: CommitCertificate, diff --git a/ethexe/malachite/core/src/store.rs b/ethexe/malachite/core/src/store.rs index f47f95725c7..109b2b2ba95 100644 --- a/ethexe/malachite/core/src/store.rs +++ b/ethexe/malachite/core/src/store.rs @@ -54,20 +54,26 @@ const META_LATEST_FINALIZED: &[u8] = b"latest_finalized"; /// Single block record kept by the service. #[derive(Clone, Encode, Decode)] pub(crate) struct BlockEntry { + /// Canonical block hash. pub block_hash: H256, + /// Parent block hash (zero for genesis). pub parent_hash: H256, + /// Block height. pub height: u64, + /// Opaque application payload. pub payload: BlockPayload, + /// Reserved envelope tail. pub reserved: [u8; 64], + /// `process_mb_proposal` has run for this block. pub saved: bool, + /// `process_mb_finalized` has run for this block. pub finalized: bool, + /// Quorum certificate, once known. pub cert: Option, } impl BlockEntry { - /// Reconstruct the [`Block`] form expected by - /// [`crate::Externalities::process_mb_proposal`] / - /// [`crate::Externalities::validate_block_above`]. + /// Reconstruct the [`Block`] form expected by [`crate::Externalities`]. pub fn block(&self) -> Block { Block { parent_hash: self.parent_hash, @@ -247,15 +253,9 @@ impl Store { } } - /// Walk back through parents from `leaf_hash` collecting every - /// ancestor that has not yet been saved. Returns `None` if the walk - /// hits a block that is not in the store (i.e. the chain is - /// incomplete and we must wait). Genesis (parent_hash == - /// `H256::zero()`) and a previously saved ancestor are valid stop - /// points. - /// - /// The returned chain is in chronological order - /// (oldest-first), ready for sequential `process_mb_proposal` calls. + /// Collect every not-yet-saved ancestor of `leaf_hash`, oldest-first. + /// Returns `None` if the walk hits a block missing from the store + /// (incomplete chain — wait for it to fill). pub fn save_chain(&self, leaf_hash: H256) -> Result>> { let mut chain_rev: Vec = Vec::new(); let mut current = leaf_hash; @@ -278,13 +278,9 @@ impl Store { Ok(Some(chain_rev)) } - /// Walk back collecting every ancestor that is `saved` but not yet - /// `finalized` and has a quorum certificate attached. Returns - /// `None` if any ancestor is missing from the store, lacks a cert, - /// or hasn't been saved (the strict invariant: finalize requires - /// save first). - /// - /// The returned chain is chronological order (oldest-first). + /// Collect every saved-but-not-finalized ancestor with a quorum cert, + /// oldest-first. Returns `None` if any ancestor is missing, lacks a + /// cert or hasn't been saved (finalize requires save first). pub fn finalize_chain(&self, leaf_hash: H256) -> Result>> { let mut chain_rev: Vec = Vec::new(); let mut current = leaf_hash; @@ -329,11 +325,9 @@ impl Store { } } - /// Drive the application's `process_mb_proposal` callback over - /// every ancestor that is now ready, starting from each seed. Cascades - /// to children: when a block becomes saveable, its descendants are - /// re-tried so a chain that was waiting on a missing middle gets - /// flushed once the gap closes. + /// Drive `save_fn` over every ancestor that is now ready, starting from + /// each seed, then cascade to children so descendants waiting on a gap + /// get flushed once it closes. pub async fn cascade_save(&self, seeds: Vec, mut save_fn: F) -> Result<()> where F: FnMut(H256, Block) -> Fut, @@ -564,11 +558,9 @@ impl Store { k } - /// Persist the engine-side `CommitCertificate` keyed by height. - /// We keep both this rich cert (with per-signer addresses, used - /// for serving sync responses) and the trimmed - /// [`crate::CommitCertificate`] inside [`BlockEntry`] (handed to - /// the application via [`crate::Externalities`]). + /// Persist the engine-side `CommitCertificate` keyed by height — + /// the rich form used for serving sync responses (the trimmed + /// [`crate::CommitCertificate`] lives inside [`BlockEntry`]). pub fn store_engine_certificate( &self, height: u64, @@ -594,10 +586,8 @@ impl Store { } } - /// Drop undecided proposals and pending parts at or below - /// `current_height`. We've already committed at this height, so - /// nothing in the engine-state columns at heights ≤ it can still be - /// reached. + /// Drop undecided proposals and pending parts at or below the already + /// committed `current_height` — they can no longer be reached. pub fn prune_engine_state(&self, current_height: u64) -> Result<()> { let mut to_delete: Vec> = Vec::new(); for p in [&[prefix::UNDECIDED][..], &[prefix::PENDING_PARTS][..]] { diff --git a/ethexe/malachite/core/src/types.rs b/ethexe/malachite/core/src/types.rs index 801ec4bd952..44c6fd40260 100644 --- a/ethexe/malachite/core/src/types.rs +++ b/ethexe/malachite/core/src/types.rs @@ -9,31 +9,16 @@ use parity_scale_codec::{Decode, Encode}; use serde::{Deserialize, Serialize}; use std::fmt::Display; -/// Hard cap on a block's encoded application payload — the SCALE-encoded -/// application operation list carried in [`Block::payload`] (the service treats -/// it as opaque bytes; the schema lives in the application crate). -/// -/// The whole [`Block`] ships as a single gossipsub message: the proposer -/// streams it as one `Data` proposal part, and the value-sync path fetches a -/// finalized block in one request-response round. Malachite's `pubsub_max_size` -/// (the gossipsub `max_transmit_size`) defaults to 4 MiB, so the encoded block -/// must stay well under that. Realistic content is ~127 KiB -/// (`MAX_INJECTED_TRANSACTIONS_SIZE_PER_MB` plus three protocol operations); the -/// 1 MiB cap leaves ~8x headroom for future operation variants while staying -/// ~4x under the 4 MiB transport ceiling (block envelope + SCALE / stream -/// framing fit comfortably in the remaining margin). +/// Hard cap on a block's encoded application payload. The whole [`Block`] +/// ships as a single gossipsub message, so the cap must stay well under +/// malachite's 4 MiB transport ceiling. pub const MAX_BLOCK_PAYLOAD_BYTES: usize = 1024 * 1024; /// Size-capped opaque application payload carried by [`Block::payload`]. pub type BlockPayload = LimitedVec; -/// 20-byte validator address. -/// -/// Newtype around [`gsigner::schemes::secp256k1::Address`] so the -/// service's API and the typical application code (ethexe today, -/// arbitrary other consumers tomorrow) share a single address shape -/// without each side reaching across crate boundaries for the inner -/// representation. +/// 20-byte validator address (newtype around +/// [`gsigner::schemes::secp256k1::Address`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] pub struct Address(pub gsigner::schemes::secp256k1::Address); @@ -63,18 +48,16 @@ impl Address { } /// Service-level block envelope: the application payload plus the -/// chain-position fields the service needs (parent hash, height) and -/// a [`Self::reserved`] tail kept for future protocol extensions. -/// -/// The block hash ([`Self::hash`]) is the [`gear_core::utils::hash`] -/// (Blake2b-256) over a SCALE-encoded -/// `(parent_hash, height, payload_hash, reserved)` tuple, where -/// `payload_hash = gear_core::utils::hash(payload.encode())`. +/// chain-position fields the service needs. #[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)] pub struct Block { + /// Hash of the parent block (zero for genesis). pub parent_hash: H256, + /// Block height (parent height + 1). pub height: u64, + /// Opaque application payload. pub payload: BlockPayload, + /// Reserved tail for future protocol extensions; currently zeroed. pub reserved: [u8; 64], } @@ -89,9 +72,8 @@ impl Block { } } - /// Compute the canonical 32-byte block hash. Deterministic — two - /// nodes with the same `(parent_hash, height, payload, reserved)` - /// produce the same hash. + /// Canonical block hash: Blake2b-256 over the SCALE-encoded + /// `(parent_hash, height, payload_hash, reserved)` tuple. pub fn hash(&self) -> H256 { let payload_bytes = self.payload.encode(); let payload_hash: H256 = gear_core::utils::hash(&payload_bytes).into(); @@ -101,14 +83,12 @@ impl Block { } /// Quorum-signed certificate proving a height was finalized. -/// -/// `signatures` is a parallel-to-validators vector of raw 64-byte -/// secp256k1 signatures (`r || s`); the application is responsible -/// for reconstructing the validator-set ordering when verifying it on -/// chain (or wherever else). #[derive(Clone, Debug, PartialEq, Eq, Encode, Decode, Serialize, Deserialize)] pub struct CommitCertificate { + /// Finalized height. pub height: u64, + /// Hash of the finalized block. pub block_hash: H256, + /// Raw 64-byte secp256k1 signatures (`r || s`), in validator-set order. pub signatures: Vec>, } diff --git a/ethexe/malachite/service/src/config.rs b/ethexe/malachite/service/src/config.rs index c380759afef..2ab6ec1809c 100644 --- a/ethexe/malachite/service/src/config.rs +++ b/ethexe/malachite/service/src/config.rs @@ -18,17 +18,13 @@ pub struct MalachiteServiceConfig { /// Gas allowance per block. pub gas_allowance: u64, - /// Number of canonical descendants an Ethereum block must have - /// before it is considered out of quarantine and safe to anchor a - /// sequencer block to. + /// Number of canonical descendants an EB must have before it is + /// out of quarantine and safe to advance to. pub canonical_quarantine: u8, - /// Extra depth (in Hoodi blocks, on top of `canonical_quarantine`) - /// the proposer waits before choosing an EB to advance to. A - /// positive value gives lagging validators time to receive the EB - /// before the proposer references it, eliminating the need for - /// validators to wait on local sync inside `validate_block_above`. - /// Defaults to 1. + /// Extra depth on top of `canonical_quarantine` the proposer waits + /// before choosing an EB to advance to, giving lagging validators + /// time to sync it. // TODO: #5478 reject unreasonable values at config load — `u32::MAX` // turns the producer's anchor walk into millions of RocksDB reads // per `wait_for_proposable_content` invocation. @@ -37,26 +33,15 @@ pub struct MalachiteServiceConfig { /// Local libp2p listen address for the Malachite swarm. pub listen_addr: SocketAddr, - /// Directory where the wrapped [`ethexe_malachite_core::MalachiteCore`] keeps - /// its WAL (`malachite/consensus.wal`) and RocksDB store - /// (`malachite/store.db/`). + /// Directory for the consensus core's WAL and RocksDB store. pub home_dir: PathBuf, - /// Multiaddrs the local node should keep persistent connections - /// to. Each entry must include the `/p2p/` suffix so the - /// swarm knows who to expect on the other side. Discovery is off, - /// so multi-validator deployments need every node listed (or at - /// least transitively reachable through the listed ones). + /// Multiaddrs of peers to keep persistent connections to; each entry + /// must include a `/p2p/` suffix (discovery is off). pub persistent_peers: Vec, - /// The complete validator set. The local node's public key (the - /// one whose secret comes from the [`gsigner::Signer`] passed to - /// [`crate::MalachiteServiceStarter::new`]) must appear in this list, or - /// service start-up fails. - /// - /// Voting power is taken at face value — Tendermint's quorum - /// threshold is `> 2/3` of the total voting power across the - /// list. + /// The complete validator set. Quorum is `> 2/3` of total voting power. + /// A validator node's own public key must appear in this list. pub validators: Vec, } @@ -64,22 +49,16 @@ impl MalachiteServiceConfig { pub const DEFAULT_GAS_ALLOWANCE: u64 = ethexe_common::DEFAULT_BLOCK_GAS_LIMIT; /// Default matches [`ethexe_common::gear::CANONICAL_QUARANTINE`]. pub const DEFAULT_CANONICAL_QUARANTINE: u8 = ethexe_common::gear::CANONICAL_QUARANTINE; - /// Default extra anchor-depth slack the proposer adds on top of - /// `canonical_quarantine`; one Hoodi block is enough to absorb the - /// typical observer-to-observer skew between validators. + /// One block is enough to absorb the typical observer skew between validators. pub const DEFAULT_POST_QUARANTINE_DELAY: u32 = 1; - /// Sits next to the typical ethexe-network 20333/udp QUIC port — - /// operators can open one contiguous range. Note the protocol - /// difference: Malachite binds a TCP listener. + /// TCP listener next to the typical ethexe-network 20333/udp QUIC port. pub const DEFAULT_LISTEN_ADDR: SocketAddr = SocketAddr::new( std::net::IpAddr::V4(std::net::Ipv4Addr::new(0, 0, 0, 0)), 20334, ); - /// Build a config with sane defaults from the node's home - /// directory. The validator set is left empty — the caller MUST - /// fill it in before passing to [`crate::MalachiteServiceStarter::new`] - /// (see [`Self::with_validators`]). + /// Build a config with defaults from the node's home directory. + /// The validator set is left empty — fill it in with [`Self::with_validators`]. pub fn from_home_dir(home_dir: PathBuf) -> Self { Self { gas_allowance: Self::DEFAULT_GAS_ALLOWANCE, @@ -129,9 +108,13 @@ impl MalachiteServiceConfig { } } +/// Validator-only part of the service configuration. pub struct ValidatorConfig { + /// Validator's on-chain public key; its secret must be in `signer`. pub pub_key: PublicKey, + /// Mempool serving injected transactions to the producer. pub mempool: M, + /// Keystore holding the validator's signing key. pub signer: Signer, } diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index 171dd585057..af5b053f64f 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -61,9 +61,14 @@ use std::{collections::VecDeque, sync::Arc}; use tokio::sync::{RwLock, mpsc::UnboundedSender}; use tracing::{debug, error, trace, warn}; +/// Constant parameters for [`EthexeExternalities`]; +/// see [`crate::MalachiteServiceConfig`] for field semantics. pub struct ExternalitiesConfig { + /// Gas allowance per block. pub gas_allowance: u64, + /// Quarantine depth an EB must clear before it can be advanced to. pub canonical_quarantine: u8, + /// Extra producer-side anchor depth on top of `canonical_quarantine`. pub post_quarantine_delay: u32, } @@ -262,8 +267,7 @@ impl Externalities for EthexeExternalities { let tx_size = tx.encoded_size(); if size_counter + tx_size > MAX_INJECTED_TRANSACTIONS_SIZE_PER_MB { - // Master's behaviour: skip oversized tx but keep - // trying smaller subsequent txs. + // Skip the oversized tx but keep trying smaller subsequent ones. continue; } diff --git a/ethexe/malachite/service/src/lib.rs b/ethexe/malachite/service/src/lib.rs index b4e1f41a1ea..7181e378c63 100644 --- a/ethexe/malachite/service/src/lib.rs +++ b/ethexe/malachite/service/src/lib.rs @@ -71,11 +71,14 @@ pub use ethexe_malachite_core::{ Multiaddr, PeerId, derive_libp2p_secret, libp2p_peer_id as malachite_libp2p_peer_id, }; -/// Ethexe-shaped commit certificate; `block_hash` is the Blake2b envelope hash. +/// Ethexe-shaped commit certificate. #[derive(Clone, Debug, PartialEq, Eq, Default)] pub struct CommitCertificate { + /// Committed MB height. pub height: u64, + /// Blake2b envelope hash of the committed MB. pub mb_hash: H256, + /// Validator signatures backing the commit. pub signatures: Vec>, } diff --git a/ethexe/malachite/service/src/mempool.rs b/ethexe/malachite/service/src/mempool.rs index ef96830b46b..f79109232ad 100644 --- a/ethexe/malachite/service/src/mempool.rs +++ b/ethexe/malachite/service/src/mempool.rs @@ -48,16 +48,9 @@ use std::{ use tokio::sync::{Notify, RwLock}; use tracing::{info, trace}; -/// Outcome of [`Mempool::insert`]. Splits into two groups: -/// -/// - **Accept** — the tx is (now or already) tracked by this validator, so -/// the caller's promise subscription remains valid. -/// - **Reject** — the tx will never be processed by this validator and -/// the caller should treat it as terminal. -/// -/// Group membership is queried via [`Self::is_accepted`]; the -/// `From for InjectedTransactionAcceptance` impl uses -/// that to project into the RPC-facing acceptance type. +/// Outcome of [`Mempool::insert`]: accept variants mean the tx is (now or +/// already) tracked by this validator; reject variants are terminal. +/// See [`Self::is_accepted`]. #[derive(Clone, Debug, PartialEq, Eq, derive_more::Display)] pub enum TxInsertionStatus { // ---- Accept ---- @@ -67,22 +60,17 @@ pub enum TxInsertionStatus { /// Same tx hash already lives in the pool — idempotent no-op. #[display("already in pool")] AlreadyInPool, - /// Same tx hash was committed within the validity window and is in - /// the seen-hash table — idempotent no-op. + /// Same tx hash was already committed within the validity window — idempotent no-op. #[display("already included within validity window")] AlreadyIncluded, // ---- Reject ---- - /// `reference_block` is past the validity window relative to the - /// latest observed head. + /// `reference_block` is past the validity window relative to the latest head. #[display("reference_block past validity window")] ExpiredRefBlock, /// Pool is at capacity. #[display("mempool at capacity")] PoolFull, - /// Per #5083, non-zero-value injected transactions are not yet - /// supported. Reject at insert so the pool never holds one — the - /// proposer cannot accidentally select it and the runtime won't - /// charge a panicking program for an out-of-budget transfer. + /// Non-zero-value injected transactions are not yet supported (#5083). #[display("non-zero value injected txs are not yet supported (#5083)")] NonZeroValue, } @@ -114,10 +102,11 @@ impl From for InjectedTransactionAcceptance { /// `forget` runs after MB finalization and dedups within `VALIDITY_WINDOW`. #[async_trait] pub trait Mempool: Send + Sync + 'static { - /// Attempt to insert a new transaction into the pool. The tx's hash is + /// Attempt to insert a new transaction into the pool. async fn insert(&self, tx: SignedInjectedTransaction) -> TxInsertionStatus; - /// Notify the pool of a new chain head. The pool uses this to evict expired + /// Notify the pool of a new chain head; evicts expired entries + /// and returns the purged transactions. async fn set_chain_head(&self, head: SimpleBlockData) -> Vec; /// Txs whose `reference_block` is an ancestor of `head`. @@ -167,9 +156,10 @@ pub const DEFAULT_POOL_CAPACITY: usize = 10_000; // `VALIDITY_WINDOW` ages them out. Needs a per-sender quota keyed on the // recovered ECDSA address. -/// Pool state behind a single mutex — operations are short, contention low. +/// Pool state behind a single lock — operations are short, contention low. #[derive(Debug, Default)] struct Inner { + /// Pending transactions by hash. pool: HashMap, SignedInjectedTransaction>, /// Recently committed txs (tx_hash → ref_block) for dedup. Aged out with the validity window. seen: HashMap, H256>, @@ -177,10 +167,14 @@ struct Inner { latest_head_height: Option, } +/// In-memory injected-tx pool backed by the node DB for ref-block resolution. #[derive(Debug)] pub struct InjectedTxMempool { + /// Mutable pool state. inner: RwLock, + /// DB for resolving `reference_block` heights and ancestor walks. db: Database, + /// Max number of pending transactions. capacity: usize, /// Notification about new transactions in `wait_for_new_tx`. new_tx_notify: Arc, @@ -302,10 +296,7 @@ impl Mempool for InjectedTxMempool { let tx_hash = tx_data.to_hash(); let ref_block = tx_data.reference_block; - // Reject non-zero-value txs unconditionally (#5083 — value-bearing - // injected txs are not supported yet). Done first so a malicious - // sender can't burn pool capacity with txs that will never be - // selectable. + // Reject non-zero-value txs first (#5083) so they never burn pool capacity. if tx_data.value != 0 { info!( %tx_hash, @@ -327,10 +318,8 @@ impl Mempool for InjectedTxMempool { return TxInsertionStatus::AlreadyInPool; } - // ref_block resolution is best-effort: a recipient that hasn't yet - // observed the producer's reference Eth block accepts and filters - // at fetch time once the block lands locally. Only reject when - // the ref_block is known AND already past the validity window. + // Unknown ref_block is accepted (filtered at fetch time); + // reject only when it is known AND already past the validity window. let ref_height_opt = self.ref_block_height(ref_block); if let Some(ref_height) = ref_height_opt && let Some(head_height) = inner.latest_head_height @@ -348,20 +337,12 @@ impl Mempool for InjectedTxMempool { return TxInsertionStatus::PoolFull; } - // Drop the lock around the DB write so concurrent inserts / - // fetches don't serialise behind disk I/O. After the write we - // re-acquire and re-run the duplicate / capacity gates: another - // concurrent insert could have landed the same tx hash, or - // filled the last capacity slot, while we were writing. + // Drop the lock around the DB write so concurrent operations don't + // serialise behind disk I/O; the gates are re-checked after re-acquire. drop(inner); // TODO: #5489 remove, set in db only after mb finalization - // Persist the tx so the local RPC's `injected_getTransactions` - // can serve it to clients that look it up by hash later. - // Done before inserting into the pool so a producer that - // immediately picks the tx is guaranteed to find it in the DB. - // The DB row is content-addressed by tx_hash, so two racing - // writes converge on the same byte content. + // Persist the tx so the local RPC can serve it by hash later. self.db.set_injected_transaction(tx.clone()); let mut inner = self.inner.write().await; @@ -387,8 +368,7 @@ impl Mempool for InjectedTxMempool { "mempool: insert accepted", ); - // Drop the lock before signaling so a waiter resumed - // immediately doesn't have to bounce on the mutex. + // Drop the lock before signaling so a resumed waiter doesn't bounce on it. drop(inner); self.new_tx_notify.notify_one(); TxInsertionStatus::Inserted @@ -438,14 +418,8 @@ impl Mempool for InjectedTxMempool { } async fn wait_for_new_tx(&self) { - // The insert path uses `notify_one`, which preserves one - // pending permit when no waiter is parked. So a tx that - // lands between the producer's `fetch()` and its `.notified()` - // call still wakes the next `.notified()` immediately. - // The caller must still re-check `fetch()` after returning — - // a permit consumed here may correspond to a tx the next - // `fetch()` already covered, in which case we just loop and - // wait again. + // `notify_one` preserves a permit when no waiter is parked, so an + // insert racing this call still wakes it. Spurious wake-ups allowed. self.new_tx_notify.notified().await } } diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index ac82cf6ec04..2458cf43370 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -1,20 +1,10 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -//! [`MalachiteService`] — public facade. +//! [`MalachiteService`] — public facade over [`ethexe_malachite_core::MalachiteCore`]. //! -//! Wraps [`ethexe_malachite_core::MalachiteService`] with the ethexe-shaped API the -//! rest of the workspace already consumes. Owns: -//! -//! - the chain-head register that [`Self::receive_new_chain_head`] -//! updates and [`crate::EthexeExternalities`] reads, -//! - the [`Mempool`] handle that serves both injected-tx routing and -//! the producer's content selection, -//! - the inner [`ethexe_malachite_core::MalachiteService`] itself, polled inline so -//! any `Err` item surfaces on this service's stream and so -//! [`Self::shutdown`] can `await` the engine actor's full teardown -//! (releasing the RocksDB advisory lock before -//! re-opening on the same home directory). +//! Routes ethexe-side inputs (chain heads, injected txs) into the consensus +//! engine and exposes its outputs as a `Stream` of [`MalachiteEvent`]s. use crate::{ MalachiteEvent, Mempool, ValidatorEntry, externalities::EthexeExternalities, @@ -40,12 +30,19 @@ use tokio::sync::mpsc::UnboundedReceiver; /// Public consensus service. pub struct MalachiteService { + /// Receiver of outbound events produced by the externalities. pub(crate) events_rx: UnboundedReceiver>, + /// Latest chain head data, shared with the externalities. pub(crate) chain_head: Arc, + /// Optional mempool for injected-tx routing; `None` when not a validator. pub(crate) mempool: Option>, + /// Externalities shared with the inner consensus core. pub(crate) externalities: Arc, + /// Known validator public keys by on-chain address, for era rotation. pub(crate) validators: HashMap, + /// Era whose validator set is currently active in the engine. pub(crate) active_era: u64, + /// Inner consensus core; `None` after shutdown. pub(crate) inner: Option>, } @@ -56,6 +53,8 @@ impl Drop for MalachiteService { } impl MalachiteService { + /// Route an injected transaction into the mempool. + /// Rejects with `PoolFull` when the node is not a validator. pub async fn receive_injected_transaction( &self, tx: SignedInjectedTransaction, @@ -67,6 +66,7 @@ impl MalachiteService { } } + /// Register a newly observed Ethereum block as the chain head. pub async fn receive_new_eb(&mut self, eb: SimpleBlockData) { let mut current = self.chain_head.latest.write().await; @@ -76,6 +76,8 @@ impl MalachiteService { } } + /// Handle a fully synced Ethereum block: publish it for the producer's + /// quarantine checks, rotate the validator set on era change and GC the mempool. pub async fn receive_eb_synced(&mut self, eb_hash: H256) { let chain_head = *self.chain_head.latest.read().await; if chain_head.hash != eb_hash { @@ -115,10 +117,13 @@ impl MalachiteService { } } + /// Handle a prepared Ethereum block: release pending events whose + /// prerequisite is now satisfied. pub async fn receive_eb_prepared(&self, _eb_hash: H256) { self.externalities.drain_pending_events().await } + /// Tear down the inner consensus core, releasing its store and sockets. pub async fn shutdown(mut self) { if let Some(inner) = self.inner.take() { inner.shutdown().await; diff --git a/ethexe/malachite/service/src/starter.rs b/ethexe/malachite/service/src/starter.rs index 23c0d753c05..45c4ad7698d 100644 --- a/ethexe/malachite/service/src/starter.rs +++ b/ethexe/malachite/service/src/starter.rs @@ -21,7 +21,8 @@ use tokio::sync::{ mpsc::{self, UnboundedReceiver}, }; -/// Consensus service starter. +/// Consensus service starter: prepares all [`MalachiteService`] components +/// up front so [`Self::start`] only has to launch the consensus core. pub struct MalachiteServiceStarter { events_rx: UnboundedReceiver>, chain_head: Arc, @@ -33,6 +34,9 @@ pub struct MalachiteServiceStarter { } impl MalachiteServiceStarter { + /// Prepare a service: resolve the node role (validator / full node), + /// build the externalities and the core config. + /// Fails if `config.validators` is empty or the validator key is absent in the signer. pub async fn new( config: MalachiteServiceConfig, validator_config: Option>, @@ -123,6 +127,7 @@ impl MalachiteServiceStarter { }) } + /// Launch the consensus core and assemble the running [`MalachiteService`]. pub async fn start(self) -> Result { let MalachiteServiceStarter { events_rx, diff --git a/ethexe/malachite/service/src/tx_validity.rs b/ethexe/malachite/service/src/tx_validity.rs index 78fe303879c..1b57f83ec4a 100644 --- a/ethexe/malachite/service/src/tx_validity.rs +++ b/ethexe/malachite/service/src/tx_validity.rs @@ -1,28 +1,11 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -//! Per-injected-tx validity, adapted from the announce-era -//! `ethexe-consensus/src/tx_validation.rs` to the Malachite Block (MB) -//! world. +//! Per-injected-tx validity checks for the Malachite Block (MB) world. //! -//! Used on both producer and validator sides so a Malachite Block whose +//! Used on both producer and validator sides so an MB whose //! `Operation::Injected(..)` payload would fail compute is rejected //! before it commits. -//! -//! Differences from master's announce-era checker: -//! -//! - The recent-included dedup walk traverses `mb_compact_block(..).parent` -//! and decodes each MB's `operations` blob (filtering for -//! [`Operation::Injected`]) instead of reading -//! `announce.injected_transactions` directly. -//! - `latest_states` is taken from the most-recent **computed** MB -//! ancestor via `mb_program_states`, walking back through -//! `mb_compact_block(..).parent` if the parent itself hasn't been -//! computed yet. -//! - The Ethereum branch walk in `is_reference_block_on_current_branch` -//! is unchanged — it still uses `block_header(..).parent_hash` -//! from the canonical Ethereum chain, fenced at -//! `globals.start_block_hash`. use anyhow::{Result, anyhow}; use ethexe_common::{ @@ -39,20 +22,13 @@ use gprimitives::{ActorId, H256}; use std::collections::HashSet; /// Minimum executable balance a destination program must have to receive -/// an injected message. Mirrors master's value: cover the panic-charge -/// floor twice over so a transient under-funding race doesn't keep -/// re-admitting a tx that will burn at execute time. -/// -/// 100 = value-per-gas. +/// an injected message: twice the panic-charge floor, at 100 value-per-gas. pub const MIN_EXECUTABLE_BALANCE_FOR_INJECTED_MESSAGES: u128 = INJECTED_MESSAGE_PANIC_GAS_CHARGE_THRESHOLD as u128 * 100 * 2; -/// Outcome of running [`TxValidityChecker::check_tx_validity`] against -/// one injected transaction. The non-`Valid` variants distinguish -/// "drop from pool" from "keep in pool, may become valid on reorg / on -/// later state changes" — the producer's mempool uses this distinction -/// to drive GC; the validator side just rejects the whole MB on any -/// non-`Valid`. +/// Outcome of [`TxValidityChecker::check_tx_validity`] for one injected +/// transaction. The producer drops non-`Valid` txs from the MB; +/// the validator rejects the whole MB on any non-`Valid`. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum TxValidity { /// Transaction is valid and can be included in an MB. @@ -75,34 +51,31 @@ pub enum TxValidity { InsufficientBalanceForInjectedMessages, } -/// Stateful checker scoped to (`chain_head`, `parent_mb`). Cache the -/// recent-included set and latest computed program states once at -/// construction; each `check_tx_validity` call is then O(VALIDITY_WINDOW) -/// for the branch walk plus a few O(1) DB lookups. +/// Stateful checker scoped to (`chain_head`, `parent_mb`); caches the +/// dedup set and the latest computed program states at construction. pub struct TxValidityChecker { + /// DB for header lookups and ancestry walks. db: Database, + /// Reference point for the validity-window and branch checks. chain_head: SimpleBlockData, + /// Local-history fence; ancestry walks stop here. start_block_hash: H256, + /// Hashes of txs included in recent MBs, for dedup. recent_included_txs: HashSet>, + /// Program states snapshot of the latest computed MB ancestor. latest_states: ProgramStates, } impl TxValidityChecker { - /// Build a checker for an MB whose parent on the consensus chain is - /// `parent_mb_hash`. Genesis maps `parent_mb_hash == H256::zero()`; the - /// zero MB is seeded as a computed ancestor by `initialize_empty_db` - /// (empty for a fresh network, the dump state for re-genesis), so its - /// `program_states` is read like any other computed snapshot. + /// Build a checker for an MB whose consensus-chain parent is + /// `parent_mb_hash` (`H256::zero()` for genesis). pub fn new_for_mb( db: Database, chain_head: SimpleBlockData, parent_mb_hash: H256, ) -> Result { - // Walk back to the most recent MB whose `meta.computed` is set — - // that's the snapshot whose `program_states` we can trust. The - // walk is bounded by the chain depth; in practice the parent - // itself is already computed because compute runs ahead of MB - // proposal. + // Walk back to the most recent computed MB — that's the snapshot + // whose `program_states` we can trust. let mut cursor = parent_mb_hash; while !cursor.is_zero() && !db.mb_meta(cursor).computed { let cb = db.mb_compact_block(cursor).ok_or_else(|| { @@ -111,8 +84,8 @@ impl TxValidityChecker { cursor = cb.parent; } - // `cursor` is either a computed MB or the zero ancestor; both carry a - // seeded `program_states` row (zero is seeded by `initialize_empty_db`). + // `cursor` is either a computed MB or the zero ancestor; both carry + // a seeded `program_states` row. let latest_states = db.mb_program_states(cursor).ok_or_else(|| { anyhow!("MB {cursor} marked computed but has no program_states row — DB invariant") })?; @@ -205,18 +178,9 @@ impl TxValidityChecker { Ok(false) } - /// Walk back `VALIDITY_WINDOW` MBs through `mb_compact_block(..).parent`, - /// decoding each MB's operations blob and harvesting the hashes - /// of every [`Operation::Injected`] for the dedup set. - /// - /// NOTE: Not bound to an instance — exposed `pub` so that - /// `[`crate::EthexeExternalities`]` can build the dedup set - /// independently of constructing a full checker. - /// - /// A missing `mb_compact_block` / `operations` row on the walk is - /// treated like reaching the start of our locally-tracked history: - /// we stop walking instead of failing. This mirrors master's - /// pragmatic break-on-missing for fast-sync recovery. + /// Collect hashes of every [`Operation::Injected`] in the last + /// `VALIDITY_WINDOW` MBs, for the dedup set. A missing MB row on the + /// walk is treated as the start of locally-tracked history. pub fn collect_recent_included_txs( db: &Database, parent_mb: H256, @@ -228,9 +192,7 @@ impl TxValidityChecker { break; } let Some(cb) = db.mb_compact_block(mb_hash) else { - // Walk fell off our locally-tracked history; stop here - // and rely on the seen-hash table inside the mempool + - // the `Outdated` rule to keep things consistent. + // Walk fell off our locally-tracked history. break; }; let Some(operations) = db.operations(cb.operations_hash) else { @@ -247,32 +209,12 @@ impl TxValidityChecker { } } -/// Programs already "touched" by Ethereum events in the open-right -/// range `(last_advanced_eb, advanced_eb]` along the canonical chain. -/// -/// Adapted from master's `block_touched_programs` which counted touched -/// programs for one EB. In the MB world an MB may span multiple EBs -/// via `AdvanceTillEthereumBlock`, so we walk every block in the range -/// (parent-walk via `block_header.parent_hash`) and accumulate. -/// -/// The set is seeded with the programs known at the latest computed -/// MB; `ProgramCreatedEvent`s along the way extend it (those new -/// actors aren't yet "touched", just known); `MirrorEvent`s on a known -/// actor count as touched. -/// -/// Returns an empty set when `advanced_eb == last_advanced_eb` (no -/// new EB to walk) or when `advanced_eb` is `H256::zero()` (no advance -/// in this MB). -/// -/// # Best-effort approximation +/// Programs "touched" by Ethereum events in the range +/// `(last_advanced_eb, advanced_eb]` along the canonical chain. +/// Returns an empty set when there is no new EB to walk. /// -/// This function is **not** a precise post-execution touched-set; it's -/// an a-priori estimate of how many programs *will* be modified during -/// the MB. Its sole job is to keep the per-MB touched-programs cap -/// honest. False positives (an event that doesn't actually modify -/// state) just make the cap stricter than necessary; false negatives -/// are bounded by `MAX_TOUCHED_PROGRAMS_PER_MB` slack at the runtime -/// layer. Do not rely on the returned set for anything beyond the cap. +/// Best-effort a-priori estimate that only serves the per-MB +/// touched-programs cap — do not rely on it for anything else. pub fn eb_touched_programs( db: &Database, last_advanced_eb: H256, @@ -282,9 +224,7 @@ pub fn eb_touched_programs( return Ok(HashSet::new()); } - // `latest_computed_mb_hash` is the zero ancestor at genesis, which - // `initialize_empty_db` seeds with the genesis / re-genesis program states - // (so under re-genesis it already lists the dump's programs as known). + // Seed the known set with the programs of the latest computed MB. let latest_computed_mb = db.globals().latest_computed_mb_hash; let mut known: HashSet = db .mb_program_states(latest_computed_mb) @@ -298,16 +238,9 @@ pub fn eb_touched_programs( .collect(); // Collect blocks in (last_advanced_eb, advanced_eb], newest-first. - // - // The walk is intentionally unbounded: `advanced_eb` has already - // passed `canonical_quarantine` (verified upstream of every caller), - // and `last_advanced_eb` is the parent MB's already-quarantine-passed - // anchor — so both points are weak-finalised on the canonical chain. - // Under any non-catastrophic reorg they share the same branch and - // the walk terminates at `last_advanced_eb` within a few EBs. - // The only divergent case is a chain reorg deeper than the - // quarantine — at that point the network has bigger problems and - // bailing at `start_block_hash` is the safe fallback. + // Both endpoints already passed quarantine, so the walk terminates + // at `last_advanced_eb` within a few EBs; the start-block fence is + // the safe fallback under a deeper-than-quarantine reorg. let mut chain = Vec::new(); let start_block_hash = db.globals().start_block_hash; let mut current = advanced_eb; @@ -317,9 +250,7 @@ pub fn eb_touched_programs( } chain.push(current); if current == start_block_hash { - // Walked back to the local start-block fence — older - // history isn't tracked. Master treats this as an error - // for `accept_announce`; we replicate by bailing. + // Local start-block fence — older history isn't tracked. break; } let header = db.block_header(current).ok_or_else(|| { @@ -328,9 +259,8 @@ pub fn eb_touched_programs( current = header.parent_hash; } - // Process oldest-first so a `ProgramCreatedEvent` populates `known` - // before any `MirrorEvent` in a later block that references that - // actor. Out-of-order would silently undercount touched programs. + // Process oldest-first so `ProgramCreatedEvent` populates `known` + // before later `MirrorEvent`s referencing that actor. chain.reverse(); let mut touched = HashSet::new(); diff --git a/ethexe/malachite/service/src/types.rs b/ethexe/malachite/service/src/types.rs index 28ddfee0b9d..6fedb8f05ea 100644 --- a/ethexe/malachite/service/src/types.rs +++ b/ethexe/malachite/service/src/types.rs @@ -4,8 +4,13 @@ use ethexe_common::SimpleBlockData; use tokio::sync::{Notify, RwLock}; +/// Ethereum chain-head register shared between [`crate::MalachiteService`] +/// (writer) and the externalities (reader). pub struct ChainHead { + /// Latest observed EB. pub latest: RwLock, + /// Latest fully synced EB — reference point for quarantine and tx checks. pub latest_synced: RwLock, + /// Wakes the producer when a new EB is synced. pub notify: Notify, } From 1aecf141f033883752f52a8a8b8bb5f0238ecee2 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 16:19:27 +0300 Subject: [PATCH 39/72] Pair fast-sync committed MB and EB --- ethexe/service/src/fast_sync.rs | 172 +++++++++++++++++++++++++++++--- 1 file changed, 158 insertions(+), 14 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index f8a4c6b0432..5fd7a358757 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -46,10 +46,15 @@ use std::{ collections::{BTreeMap, BTreeSet, HashMap}, }; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct CommittedChain { + pub mb_hash: H256, + pub eb_hash: H256, +} + struct EventData { latest_committed_batch: Digest, - latest_committed_mb: H256, - latest_committed_eb: Option, + latest_committed_chain: CommittedChain, } impl EventData { @@ -59,13 +64,13 @@ impl EventData { highest_block: H256, ) -> Result> { let mut latest_committed_batch = None; - let mut latest_committed_mb = None; - let mut latest_committed_eb = None; + let mut latest_committed_chain = None; let mut block = highest_block; while !db.block_meta(block).prepared { let block_data = block_loader.load(block, None).await?; + let mut pending_eb = None; for event in block_data.events.iter().rev() { match event { BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { @@ -73,30 +78,36 @@ impl EventData { })) => { latest_committed_batch.get_or_insert(*digest); } - BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb_hash))) => { - latest_committed_mb.get_or_insert(*mb_hash); - } BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(eb_hash))) => { - latest_committed_eb.get_or_insert(*eb_hash); + pending_eb = Some(*eb_hash); + } + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb_hash))) => { + latest_committed_chain.get_or_insert(CommittedChain { + mb_hash: *mb_hash, + eb_hash: pending_eb.take().unwrap_or_default(), + }); } _ => {} } } + if latest_committed_chain.is_some() { + break; + } + block = block_data.header.parent_hash; if block.is_zero() { break; } } - let Some(latest_committed_mb) = latest_committed_mb else { + let Some(latest_committed_chain) = latest_committed_chain else { return Ok(None); }; Ok(Some(Self { latest_committed_batch: latest_committed_batch.unwrap_or_default(), - latest_committed_mb, - latest_committed_eb, + latest_committed_chain, })) } } @@ -651,11 +662,15 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { let EventData { latest_committed_batch, - latest_committed_mb, - latest_committed_eb, + latest_committed_chain, } = event_data; - let latest_committed_eb = latest_committed_eb.unwrap_or(genesis_block_hash); + let latest_committed_mb = latest_committed_chain.mb_hash; + let latest_committed_eb = if latest_committed_chain.eb_hash.is_zero() { + genesis_block_hash + } else { + latest_committed_chain.eb_hash + }; let block_data = block_loader.load(latest_committed_eb, None).await?; let code_ids = collect_code_ids( @@ -749,3 +764,132 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use ethexe_common::{BlockHeader, SimpleBlockData}; + use ethexe_ethereum::IntoBlockId; + use std::{ + collections::{BTreeMap, HashMap}, + ops::RangeInclusive, + }; + + #[derive(Default)] + struct TestBlockLoader { + blocks: BTreeMap, + } + + impl BlockLoader for TestBlockLoader { + async fn load_simple(&self, _block: impl IntoBlockId) -> Result { + anyhow::bail!("load_simple is not used by EventData::collect tests") + } + + async fn load(&self, hash: H256, _header: Option) -> Result { + self.blocks + .get(&hash) + .cloned() + .with_context(|| format!("missing test block {hash}")) + } + + async fn load_many(&self, _range: RangeInclusive) -> Result> { + anyhow::bail!("load_many is not used by EventData::collect tests") + } + } + + fn test_block(hash: H256, parent_hash: H256, events: Vec) -> BlockData { + BlockData { + hash, + header: BlockHeader { + height: hash.to_low_u64_be() as u32, + timestamp: 0, + parent_hash, + }, + events, + } + } + + #[tokio::test] + async fn event_data_collects_committed_chain_as_single_pair() { + let db = Database::memory(); + let older_block = H256::from_low_u64_be(1); + let newer_block = H256::from_low_u64_be(2); + let older_mb = H256::repeat_byte(0x11); + let older_eb = H256::repeat_byte(0x22); + let newer_mb = H256::repeat_byte(0x33); + let newer_eb = H256::repeat_byte(0x44); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + newer_block, + test_block( + newer_block, + older_block, + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(newer_mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(newer_eb))), + ], + ), + ); + loader.blocks.insert( + older_block, + test_block( + older_block, + H256::zero(), + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(older_mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(older_eb))), + ], + ), + ); + + let event_data = EventData::collect(&loader, &db, newer_block) + .await + .unwrap() + .expect("committed chain found"); + + assert_eq!(event_data.latest_committed_chain.mb_hash, newer_mb); + assert_eq!(event_data.latest_committed_chain.eb_hash, newer_eb); + } + + #[tokio::test] + async fn event_data_does_not_mix_newer_mb_with_older_eb() { + let db = Database::memory(); + let older_block = H256::from_low_u64_be(1); + let newer_block = H256::from_low_u64_be(2); + let older_mb = H256::repeat_byte(0x11); + let older_eb = H256::repeat_byte(0x22); + let newer_mb = H256::repeat_byte(0x33); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + newer_block, + test_block( + newer_block, + older_block, + vec![BlockEvent::Router(RouterEvent::MBCommitted( + MBCommittedEvent(newer_mb), + ))], + ), + ); + loader.blocks.insert( + older_block, + test_block( + older_block, + H256::zero(), + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(older_mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(older_eb))), + ], + ), + ); + + let event_data = EventData::collect(&loader, &db, newer_block) + .await + .unwrap() + .expect("committed chain found"); + + assert_eq!(event_data.latest_committed_chain.mb_hash, newer_mb); + assert_eq!(event_data.latest_committed_chain.eb_hash, H256::zero()); + } +} From 792c897c7d6df56202bb7ac5b0d493718d52a57a Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 16:55:07 +0300 Subject: [PATCH 40/72] Filter fast-sync replay events locally --- ethexe/malachite/service/src/externalities.rs | 606 +++++++----------- ethexe/malachite/service/src/service.rs | 12 +- ethexe/service/src/fast_sync.rs | 7 +- 3 files changed, 225 insertions(+), 400 deletions(-) diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index 10ac67ed6ef..d3bcf06d99d 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -63,15 +63,16 @@ use std::{ use tokio::sync::{Notify, mpsc}; use tracing::{error, info, warn}; -#[derive(Clone, Copy)] -pub(crate) enum ReplayBoundary { - /// No replay boundary configured (both hashes are zero). - Disabled, - /// Stable replay suppression applies for MBs at or below this height. - AtHeight(u64), - /// Boundary hash is known, but compact is not yet available. - /// Suppress replay callbacks until the boundary MB itself is observed. - AtHash(H256), +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct FastSyncReplayBoundary { + pub mb_hash: H256, + pub eb_hash: H256, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct FastSyncReplayFilter { + pub boundary: FastSyncReplayBoundary, + pub boundary_proposal_seen: bool, } /// Inputs the externalities need to satisfy the [`ethexe_malachite_core::Externalities`] @@ -113,9 +114,11 @@ pub(crate) struct EthexeExternalities { /// time they see the MB. Validators do NOT apply this depth — they accept /// any advance at depth ≥ `canonical_quarantine`. pub(crate) post_quarantine_delay: u32, - /// Captured once per externalities instance; prevents mutable live - /// computed progression from changing replay suppression scope. - pub(crate) replay_boundary: RwLock, + /// Fast-sync startup gate, not a general live/replay classifier. + /// While active it suppresses all Malachite callbacks until the + /// captured boundary proposal and finalization are replayed. Callers + /// must enable it before starting the Malachite application task. + pub(crate) fast_sync_replay_filter: RwLock>, } /// One outbound [`MalachiteEvent`] that can't be released until its @@ -135,7 +138,7 @@ impl Externalities for EthexeExternalities { &self, mb_hash: H256, mb: Block, - origin: CallbackOrigin, + _origin: CallbackOrigin, ) -> Result<()> { // Runs on the proposer, participant, and sync paths. Frozen // discriminants mean every historical operation always decodes; an @@ -183,9 +186,7 @@ impl Externalities for EthexeExternalities { meta.last_advanced_eb = last_advanced; }); - if origin == CallbackOrigin::ValueSyncReplay - && self.is_replay_mb_below_or_at_execution_boundary(mb_hash, mb.height) - { + if self.suppress_for_fast_sync_replay_proposal(mb_hash, last_advanced)? { return Ok(()); } @@ -203,7 +204,7 @@ impl Externalities for EthexeExternalities { &self, mb_hash: H256, cert: ethexe_malachite_core::CommitCertificate, - origin: CallbackOrigin, + _origin: CallbackOrigin, ) -> Result<()> { let compact = self.db.mb_compact_block(mb_hash).ok_or_else(|| { anyhow!( @@ -218,9 +219,7 @@ impl Externalities for EthexeExternalities { ) })?; - if origin == CallbackOrigin::ValueSyncReplay - && self.is_replay_mb_below_or_at_execution_boundary(mb_hash, compact.height) - { + if self.suppress_for_fast_sync_replay_finalization(mb_hash) { return Ok(()); } @@ -646,70 +645,70 @@ impl Externalities for EthexeExternalities { } impl EthexeExternalities { - /// Whether `mb_hash`/`mb_height` are at or below the execution replay - /// boundary. - /// - /// Capture once per externalities instance and keep stable even if - /// live compute advances globals in between replay callbacks. - fn is_replay_mb_below_or_at_execution_boundary(&self, mb_hash: H256, mb_height: u64) -> bool { - let mut replay_boundary = self - .replay_boundary + pub(crate) fn enable_fast_sync_replay_filter(&self, mb_hash: H256, eb_hash: H256) { + *self + .fast_sync_replay_filter .write() - .expect("replay_boundary poisoned"); - - match *replay_boundary { - ReplayBoundary::Disabled => false, - ReplayBoundary::AtHeight(height) => mb_height <= height, - ReplayBoundary::AtHash(boundary_hash) => { - if mb_hash != boundary_hash { - return true; - } - - if let Some(boundary_compact) = self.db.mb_compact_block(boundary_hash) { - *replay_boundary = ReplayBoundary::AtHeight(boundary_compact.height); - return mb_height <= boundary_compact.height; - } - - true - } - } + .expect("fast_sync_replay_filter poisoned") = Some(FastSyncReplayFilter { + boundary: FastSyncReplayBoundary { mb_hash, eb_hash }, + boundary_proposal_seen: false, + }); } - /// Capture a replay boundary from DB globals, preferring - /// `latest_computed_mb_hash` and falling back to - /// `latest_finalized_mb_hash`. - /// - /// This method intentionally does not mutate globals and does not - /// infer a boundary from any other signal. If no execution boundary - /// is available, suppression stays disabled. - fn read_replay_boundary_from_db(&self) -> ReplayBoundary { - let boundary_hash = { - let globals = self.db.globals(); - if !globals.latest_computed_mb_hash.is_zero() { - globals.latest_computed_mb_hash - } else { - globals.latest_finalized_mb_hash - } + fn suppress_for_fast_sync_replay_proposal( + &self, + mb_hash: H256, + last_advanced_eb: H256, + ) -> Result { + let mut guard = self + .fast_sync_replay_filter + .write() + .expect("fast_sync_replay_filter poisoned"); + + let Some(filter) = *guard else { + return Ok(false); }; - if boundary_hash.is_zero() { - return ReplayBoundary::Disabled; + let boundary = filter.boundary; + let matches_boundary = mb_hash == boundary.mb_hash && last_advanced_eb == boundary.eb_hash; + + if mb_hash == boundary.mb_hash && !matches_boundary { + let message = format!( + "fast-sync replay boundary mismatch: MB {mb_hash} advanced {last_advanced_eb}, expected {}", + boundary.eb_hash, + ); + *guard = None; + drop(guard); + + let _ = self.event_tx.send(Err(anyhow!(message.clone()))); + return Err(anyhow!(message)); } - self.db.mb_compact_block(boundary_hash).map_or_else( - || ReplayBoundary::AtHash(boundary_hash), - |boundary_compact| ReplayBoundary::AtHeight(boundary_compact.height), - ) + if matches_boundary { + guard + .as_mut() + .expect("fast-sync replay filter checked above") + .boundary_proposal_seen = true; + } + + Ok(true) } - /// Enable replay-boundary suppression from the current execution snapshot - /// persisted in DB globals. - pub(crate) fn enable_replay_boundary_from_execution_snapshot(&self) { - let replay_boundary = self.read_replay_boundary_from_db(); - *self - .replay_boundary + fn suppress_for_fast_sync_replay_finalization(&self, mb_hash: H256) -> bool { + let mut guard = self + .fast_sync_replay_filter .write() - .expect("replay_boundary poisoned") = replay_boundary; + .expect("fast_sync_replay_filter poisoned"); + + let Some(filter) = *guard else { + return false; + }; + + if mb_hash == filter.boundary.mb_hash && filter.boundary_proposal_seen { + *guard = None; + } + + true } /// True iff `prerequisite.is_zero()` (no prerequisite — genesis @@ -840,7 +839,7 @@ mod tests { use crate::{MalachiteEvent, mempool::EmptyMempool}; use anyhow::Context; use ethexe_common::{ - BlockHeader, DEFAULT_BLOCK_GAS_LIMIT, + BlockHeader, db::{BlockMetaStorageRW, OnChainStorageRW}, injected::PurgedTransaction, }; @@ -890,7 +889,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 0, post_quarantine_delay: 0, - replay_boundary: RwLock::new(ReplayBoundary::Disabled), + fast_sync_replay_filter: RwLock::new(None), }; (ext, event_rx) } @@ -1320,7 +1319,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 0, post_quarantine_delay: 0, - replay_boundary: RwLock::new(ReplayBoundary::Disabled), + fast_sync_replay_filter: RwLock::new(None), }; let payload = Operations::new(vec![ @@ -1411,24 +1410,18 @@ mod tests { assert_eq!(db.globals().latest_finalized_mb_hash, replay_hash); } - /// MBs at or below the execution boundary must persist MB - /// identity rows but never emit service events or retreat finalized - /// globals. #[tokio::test] - async fn replay_below_execution_boundary_does_not_emit_or_retreat_globals() { + async fn fast_sync_filter_suppresses_until_boundary_finalization() { use ethexe_common::db::MbStorageRO; let db = Database::memory(); let (ext, mut rx) = make_externalities(db.clone()); - let boundary_payload = payload(None, 1); + let boundary_eb = H256::repeat_byte(0xA1); + let boundary_payload = payload(Some(boundary_eb), 1); let boundary_block = wrap(boundary_payload, 10, H256::zero()); let boundary_hash = boundary_block.hash(); - db.globals_mutate(|g| { - g.latest_finalized_mb_hash = boundary_hash; - g.latest_computed_mb_hash = boundary_hash; - }); - ext.enable_replay_boundary_from_execution_snapshot(); + ext.enable_fast_sync_replay_filter(boundary_hash, boundary_eb); ext.process_mb_proposal( boundary_hash, @@ -1437,132 +1430,52 @@ mod tests { ) .await .unwrap(); - assert!( - rx.try_recv().is_err(), - "boundary replay should be suppressed" - ); + assert!(rx.try_recv().is_err(), "boundary proposal is suppressed"); assert!( ext.pending_events .lock() .expect("pending_events poisoned") - .is_empty(), - "boundary replay should not queue events", + .is_empty() ); + assert_eq!(db.mb_meta(boundary_hash).last_advanced_eb, boundary_eb); + assert!(db.mb_compact_block(boundary_hash).is_some()); - let old_eb = H256::repeat_byte(0xA1); - let replay_payload = Operations::new(vec![ - Operation::AdvanceTillEthereumBlock { block_hash: old_eb }, - Operation::ProgressTasks, - Operation::ProcessQueues { - gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, - }, - ]); - let replay_block = wrap(replay_payload, 10, H256::zero()); - let replay_hash = replay_block.hash(); - - ext.process_mb_proposal(replay_hash, replay_block, CallbackOrigin::ValueSyncReplay) - .await - .unwrap(); - assert!(rx.try_recv().is_err(), "proposal replay must not emit"); - assert!( - ext.pending_events - .lock() - .expect("pending_events poisoned") - .is_empty(), - "old replay must not queue service events", - ); - - assert_eq!(db.mb_meta(replay_hash).last_advanced_eb, old_eb); - let replay_compact = db - .mb_compact_block(replay_hash) - .expect("replay compact persisted"); - assert_eq!(replay_compact.height, 10); - assert!(db.operations(replay_compact.operations_hash).is_some()); - - ext.process_mb_finalized(replay_hash, fake_cert(10), CallbackOrigin::ValueSyncReplay) - .await - .unwrap(); - assert_eq!(db.globals().latest_finalized_mb_hash, boundary_hash); - assert!(rx.try_recv().is_err(), "finalization replay must not emit"); - assert!( - ext.pending_events - .lock() - .expect("pending_events poisoned") - .is_empty(), - "old replay must not enqueue pending service events", - ); - } - - /// Post-boundary MBs should keep normal service event behavior. - #[tokio::test] - async fn post_boundary_mb_emits_events_normally() { - let db = Database::memory(); - let (ext, mut rx) = make_externalities(db.clone()); - - let boundary_payload = payload(None, 1); - let boundary_block = wrap(boundary_payload, 10, H256::zero()); - let boundary_hash = boundary_block.hash(); - db.globals_mutate(|g| { - g.latest_finalized_mb_hash = boundary_hash; - g.latest_computed_mb_hash = boundary_hash; - }); - ext.enable_replay_boundary_from_execution_snapshot(); - - ext.process_mb_proposal( + ext.process_mb_finalized( boundary_hash, - boundary_block, + fake_cert(10), CallbackOrigin::ValueSyncReplay, ) .await .unwrap(); assert!( rx.try_recv().is_err(), - "boundary replay should be suppressed" + "boundary finalization is suppressed" ); assert!( ext.pending_events .lock() .expect("pending_events poisoned") - .is_empty(), - "boundary replay should not queue events", + .is_empty() ); - let advanced_eb = H256::repeat_byte(0xA2); - db.mutate_block_meta(advanced_eb, |meta| { - meta.prepared = true; - }); + let post_eb = H256::repeat_byte(0xA2); + db.mutate_block_meta(post_eb, |meta| meta.prepared = true); + let post_payload = payload(Some(post_eb), 2); + let post_block = wrap(post_payload, 11, boundary_hash); + let post_hash = post_block.hash(); - let replay_payload = Operations::new(vec![ - Operation::AdvanceTillEthereumBlock { - block_hash: advanced_eb, - }, - Operation::ProgressTasks, - Operation::ProcessQueues { - gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, - }, - ]); - let replay_block = wrap(replay_payload, 11, boundary_hash); - let replay_hash = replay_block.hash(); - - ext.process_mb_proposal(replay_hash, replay_block, CallbackOrigin::ValueSyncReplay) + ext.process_mb_proposal(post_hash, post_block, CallbackOrigin::ValueSyncReplay) .await .unwrap(); - assert_eq!( - ext.pending_events - .lock() - .expect("pending_events poisoned") - .len(), - 0 - ); match rx.recv().await.expect("proposal event").unwrap() { MalachiteEvent::BlockProposal { height, mb_hash } => { assert_eq!(height, 11); - assert_eq!(mb_hash, replay_hash); + assert_eq!(mb_hash, post_hash); } other => panic!("expected BlockProposal, got {other:?}"), } - ext.process_mb_finalized(replay_hash, fake_cert(11), CallbackOrigin::ValueSyncReplay) + ext.process_mb_finalized(post_hash, fake_cert(11), CallbackOrigin::ValueSyncReplay) .await .unwrap(); match rx.recv().await.expect("finalization event").unwrap() { @@ -1571,270 +1484,181 @@ mod tests { height, mb_hash, } => { - assert_eq!(height, 11); assert_eq!(cert.height, 11); - assert_eq!(mb_hash, replay_hash); + assert_eq!(height, 11); + assert_eq!(mb_hash, post_hash); } other => panic!("expected BlockFinalized, got {other:?}"), } - assert_eq!(db.globals().latest_finalized_mb_hash, replay_hash); + assert_eq!(db.globals().latest_finalized_mb_hash, post_hash); } - /// If the execution boundary compact is still missing, suppress replay - /// callbacks by hash until the boundary MB is observed. #[tokio::test] - async fn replay_with_missing_boundary_compact_suppresses_until_boundary_is_seen() { + async fn fast_sync_filter_retains_until_boundary_proposal_seen() { let db = Database::memory(); let (ext, mut rx) = make_externalities(db.clone()); - let boundary_payload = payload(None, 2); - let boundary_block = wrap(boundary_payload, 10, H256::zero()); + let boundary_eb = H256::repeat_byte(0xD1); + let boundary_payload = payload(Some(boundary_eb), 1); + let boundary_block = wrap(boundary_payload.clone(), 10, H256::zero()); let boundary_hash = boundary_block.hash(); + ext.enable_fast_sync_replay_filter(boundary_hash, boundary_eb); - db.globals_mutate(|g| { - g.latest_finalized_mb_hash = boundary_hash; - g.latest_computed_mb_hash = boundary_hash; - }); - ext.enable_replay_boundary_from_execution_snapshot(); - - let old_eb = H256::repeat_byte(0xB1); - let old_payload = Operations::new(vec![ - Operation::AdvanceTillEthereumBlock { block_hash: old_eb }, - Operation::ProgressTasks, - Operation::ProcessQueues { - gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, + let operations_hash = db.set_operations(boundary_payload); + db.set_mb_compact_block( + boundary_hash, + CompactMb { + parent: H256::zero(), + height: 10, + operations_hash, }, - ]); - let old_block = wrap(old_payload, 9, H256::zero()); - let old_hash = old_block.hash(); - - ext.process_mb_proposal(old_hash, old_block, CallbackOrigin::ValueSyncReplay) - .await - .unwrap(); - ext.process_mb_finalized(old_hash, fake_cert(9), CallbackOrigin::ValueSyncReplay) - .await - .unwrap(); + ); - assert!(rx.try_recv().is_err(), "replay should be suppressed"); + ext.process_mb_finalized( + boundary_hash, + fake_cert(10), + CallbackOrigin::ValueSyncReplay, + ) + .await + .unwrap(); assert!( - ext.pending_events - .lock() - .expect("pending_events poisoned") - .is_empty(), - "replay should not enqueue pending service events", + rx.try_recv().is_err(), + "boundary finalization before proposal is suppressed" ); - assert_eq!( - db.globals().latest_finalized_mb_hash, - boundary_hash, - "replay finalization must not retreat finalized boundary", + assert!( + ext.fast_sync_replay_filter + .read() + .expect("fast_sync_replay_filter poisoned") + .is_some(), + "filter must stay active until boundary proposal is seen", ); - // Boundary MB itself is still missing in compact storage when replay starts. - // Process it now to both persist consensus identity and switch boundary mode. + let retained_eb = H256::repeat_byte(0xD2); + db.mutate_block_meta(retained_eb, |meta| meta.prepared = true); + let retained_block = wrap(payload(Some(retained_eb), 2), 11, boundary_hash); + let retained_hash = retained_block.hash(); ext.process_mb_proposal( - boundary_hash, - boundary_block, + retained_hash, + retained_block, CallbackOrigin::ValueSyncReplay, ) .await .unwrap(); assert!( rx.try_recv().is_err(), - "boundary MB is still replay context" + "non-boundary proposal is still suppressed while filter is retained" ); - let advanced_eb = H256::repeat_byte(0xB2); - db.mutate_block_meta(advanced_eb, |meta| { - meta.prepared = true; - }); - - let post_boundary_payload = Operations::new(vec![ - Operation::AdvanceTillEthereumBlock { - block_hash: advanced_eb, - }, - Operation::ProgressTasks, - Operation::ProcessQueues { - gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, - }, - ]); - let post_boundary_block = wrap(post_boundary_payload, 11, boundary_hash); - let post_boundary_hash = post_boundary_block.hash(); - ext.process_mb_proposal( - post_boundary_hash, - post_boundary_block, + boundary_hash, + boundary_block, CallbackOrigin::ValueSyncReplay, ) .await .unwrap(); - match rx.recv().await.expect("proposal event").unwrap() { - MalachiteEvent::BlockProposal { height, mb_hash } => { - assert_eq!(height, 11); - assert_eq!(mb_hash, post_boundary_hash); - } - other => panic!("expected BlockProposal, got {other:?}"), - } - ext.process_mb_finalized( - post_boundary_hash, - fake_cert(11), + boundary_hash, + fake_cert(10), CallbackOrigin::ValueSyncReplay, ) .await .unwrap(); - match rx.recv().await.expect("finalization event").unwrap() { - MalachiteEvent::BlockFinalized { - height, mb_hash, .. - } => { - assert_eq!(height, 11); - assert_eq!(mb_hash, post_boundary_hash); - } - other => panic!("expected BlockFinalized, got {other:?}"), - } - assert_eq!(db.globals().latest_finalized_mb_hash, post_boundary_hash); - } - - /// When the boundary compact is missing, old replay callbacks are - /// still suppressed, but live callbacks for post-boundary MBs must - /// continue to emit and advance globals. - #[tokio::test] - async fn live_callbacks_not_suppressed_when_boundary_compact_is_missing() { - let db = Database::memory(); - let (ext, mut rx) = make_externalities(db.clone()); - - let boundary_payload = payload(None, 2); - let boundary_block = wrap(boundary_payload, 10, H256::zero()); - let boundary_hash = boundary_block.hash(); - db.globals_mutate(|g| { - g.latest_finalized_mb_hash = boundary_hash; - g.latest_computed_mb_hash = boundary_hash; - }); - ext.enable_replay_boundary_from_execution_snapshot(); - - let replay_payload = Operations::new(vec![ - Operation::ProgressTasks, - Operation::ProcessQueues { - gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, - }, - ]); - let old_block = wrap(replay_payload, 9, H256::zero()); - let old_hash = old_block.hash(); - ext.process_mb_proposal(old_hash, old_block, CallbackOrigin::ValueSyncReplay) - .await - .unwrap(); - assert!(rx.try_recv().is_err(), "replay should be suppressed"); + assert!( + ext.fast_sync_replay_filter + .read() + .expect("fast_sync_replay_filter poisoned") + .is_none(), + "boundary proposal plus finalization clears filter", + ); - let live_payload = Operations::new(vec![ - Operation::ProgressTasks, - Operation::ProcessQueues { - gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, - }, - ]); - let live_block = wrap(live_payload, 11, boundary_hash); - let live_hash = live_block.hash(); - ext.process_mb_proposal(live_hash, live_block, CallbackOrigin::Live) + let post_eb = H256::repeat_byte(0xD3); + db.mutate_block_meta(post_eb, |meta| meta.prepared = true); + let post_block = wrap(payload(Some(post_eb), 3), 12, boundary_hash); + let post_hash = post_block.hash(); + ext.process_mb_proposal(post_hash, post_block, CallbackOrigin::ValueSyncReplay) .await .unwrap(); match rx.recv().await.expect("proposal event").unwrap() { MalachiteEvent::BlockProposal { height, mb_hash } => { - assert_eq!(height, 11); - assert_eq!(mb_hash, live_hash); + assert_eq!(height, 12); + assert_eq!(mb_hash, post_hash); } other => panic!("expected BlockProposal, got {other:?}"), } - - ext.process_mb_finalized(live_hash, fake_cert(11), CallbackOrigin::Live) - .await - .unwrap(); - match rx.recv().await.expect("finalization event").unwrap() { - MalachiteEvent::BlockFinalized { - cert, - height, - mb_hash, - } => { - assert_eq!(height, 11); - assert_eq!(cert.height, 11); - assert_eq!(mb_hash, live_hash); - } - other => panic!("expected BlockFinalized, got {other:?}"), - } - assert_eq!(db.globals().latest_finalized_mb_hash, live_hash); } - /// Live recomputation progress must not make the replay boundary - /// move forward before finalization processing. #[tokio::test] - async fn computed_tip_advancing_does_not_disable_post_boundary_finalization_events() { + async fn fast_sync_filter_rejects_boundary_eb_mismatch() { let db = Database::memory(); - let (ext, mut rx) = make_externalities(db.clone()); + let (ext, mut rx) = make_externalities(db); - let boundary_payload = payload(None, 1); - let boundary_block = wrap(boundary_payload, 10, H256::zero()); - let boundary_hash = boundary_block.hash(); - db.globals_mutate(|g| { - g.latest_finalized_mb_hash = boundary_hash; - g.latest_computed_mb_hash = boundary_hash; - }); - ext.enable_replay_boundary_from_execution_snapshot(); + let expected_eb = H256::repeat_byte(0xB1); + let actual_eb = H256::repeat_byte(0xB2); + let block = wrap(payload(Some(actual_eb), 1), 10, H256::zero()); + let mb_hash = block.hash(); + ext.enable_fast_sync_replay_filter(mb_hash, expected_eb); - ext.process_mb_proposal( - boundary_hash, - boundary_block, - CallbackOrigin::ValueSyncReplay, - ) - .await - .unwrap(); + let err = ext + .process_mb_proposal(mb_hash, block, CallbackOrigin::ValueSyncReplay) + .await + .expect_err("boundary EB mismatch must fail"); assert!( - rx.try_recv().is_err(), - "boundary replay should be suppressed" + err.to_string() + .contains("fast-sync replay boundary mismatch"), + "unexpected error: {err:#}" ); - - let advanced_eb = H256::repeat_byte(0xA3); - db.mutate_block_meta(advanced_eb, |meta| { - meta.prepared = true; - }); - - let replay_payload = Operations::new(vec![ - Operation::AdvanceTillEthereumBlock { - block_hash: advanced_eb, - }, - Operation::ProgressTasks, - Operation::ProcessQueues { - gas_allowance: DEFAULT_BLOCK_GAS_LIMIT, - }, - ]); - let replay_block = wrap(replay_payload, 11, boundary_hash); - let replay_hash = replay_block.hash(); - - ext.process_mb_proposal(replay_hash, replay_block, CallbackOrigin::ValueSyncReplay) - .await - .unwrap(); - match rx.recv().await.expect("proposal event").unwrap() { - MalachiteEvent::BlockProposal { height, mb_hash } => { - assert_eq!(height, 11); - assert_eq!(mb_hash, replay_hash); - } - other => panic!("expected BlockProposal, got {other:?}"), + match rx.try_recv().expect("mismatch event") { + Err(err) => assert!( + err.to_string() + .contains("fast-sync replay boundary mismatch"), + "unexpected event error: {err:#}" + ), + Ok(event) => panic!("expected mismatch error event, got {event:?}"), } + assert!( + ext.fast_sync_replay_filter + .read() + .expect("fast_sync_replay_filter poisoned") + .is_none(), + "mismatch must clear replay filter", + ); + } + + #[tokio::test] + async fn fast_sync_filter_rejects_zero_boundary_eb_mismatch() { + let db = Database::memory(); + let (ext, mut rx) = make_externalities(db); - db.globals_mutate(|g| g.latest_computed_mb_hash = replay_hash); + let expected_eb = H256::zero(); + let actual_eb = H256::repeat_byte(0xC1); + let block = wrap(payload(Some(actual_eb), 1), 10, H256::zero()); + let mb_hash = block.hash(); + ext.enable_fast_sync_replay_filter(mb_hash, expected_eb); - ext.process_mb_finalized(replay_hash, fake_cert(11), CallbackOrigin::ValueSyncReplay) + let err = ext + .process_mb_proposal(mb_hash, block, CallbackOrigin::ValueSyncReplay) .await - .unwrap(); - match rx.recv().await.expect("finalization event").unwrap() { - MalachiteEvent::BlockFinalized { - cert, - height, - mb_hash, - } => { - assert_eq!(height, 11); - assert_eq!(cert.height, 11); - assert_eq!(mb_hash, replay_hash); - } - other => panic!("expected BlockFinalized, got {other:?}"), + .expect_err("zero boundary EB mismatch must fail"); + assert!( + err.to_string() + .contains("fast-sync replay boundary mismatch"), + "unexpected error: {err:#}" + ); + match rx.try_recv().expect("mismatch event") { + Err(err) => assert!( + err.to_string() + .contains("fast-sync replay boundary mismatch"), + "unexpected event error: {err:#}" + ), + Ok(event) => panic!("expected mismatch error event, got {event:?}"), } - assert_eq!(db.globals().latest_finalized_mb_hash, replay_hash); + assert!( + ext.fast_sync_replay_filter + .read() + .expect("fast_sync_replay_filter poisoned") + .is_none(), + "mismatch must clear replay filter", + ); } // ------------------------------------------------------------------ @@ -1864,7 +1688,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 0, post_quarantine_delay: 0, - replay_boundary: RwLock::new(ReplayBoundary::Disabled), + fast_sync_replay_filter: RwLock::new(None), }; (ext, event_rx) } @@ -2616,7 +2440,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 2, post_quarantine_delay: 3, - replay_boundary: RwLock::new(ReplayBoundary::Disabled), + fast_sync_replay_filter: RwLock::new(None), }; let candidate = ext diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 9844eafdd20..d81e0eb4ca3 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -36,8 +36,7 @@ use gsigner::{Signer, schemes::secp256k1::Secp256k1}; use tokio::sync::{Notify, mpsc}; use crate::{ - MalachiteConfig, MalachiteEvent, Mempool, ValidatorEntry, - externalities::{EthexeExternalities, ReplayBoundary}, + MalachiteConfig, MalachiteEvent, Mempool, ValidatorEntry, externalities::EthexeExternalities, }; /// Public consensus service. @@ -166,7 +165,7 @@ impl MalachiteService { gas_allowance: config.gas_allowance, canonical_quarantine: config.canonical_quarantine, post_quarantine_delay: config.post_quarantine_delay, - replay_boundary: std::sync::RwLock::new(ReplayBoundary::Disabled), + fast_sync_replay_filter: std::sync::RwLock::new(None), }); // On-chain addresses → pub keys, so era rotations resolve back without an out-of-band lookup. @@ -199,12 +198,9 @@ impl MalachiteService { } } - /// Capture the execution-layer replay boundary from DB globals so - /// `ValueSyncReplay` callbacks can be suppressed for pre-boundary - /// Malachite MBs after fast-sync. - pub fn enable_replay_boundary_from_execution_snapshot(&self) { + pub fn enable_fast_sync_replay_filter(&self, mb_hash: H256, eb_hash: H256) { self.externalities - .enable_replay_boundary_from_execution_snapshot(); + .enable_fast_sync_replay_filter(mb_hash, eb_hash); } /// Hand an injected transaction to the mempool. The local diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 5fd7a358757..62e63c8d5d5 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -745,7 +745,12 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { }); if let Some(malachite) = malachite.as_mut() { - malachite.enable_replay_boundary_from_execution_snapshot(); + // `Service::run` performs fast sync before `run_inner().start_app_task()`, + // so live Malachite callbacks cannot race this startup replay gate. + malachite.enable_fast_sync_replay_filter( + latest_committed_chain.mb_hash, + latest_committed_chain.eb_hash, + ); malachite.receive_new_chain_head(block_data.to_simple()); } From 91a9778bbd48b4a21417a1c8c8d2dca2c6871bd3 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 17:51:15 +0300 Subject: [PATCH 41/72] Remove Malachite callback origin plumbing --- ethexe/malachite/core/src/app.rs | 195 ++---------------- ethexe/malachite/core/src/externalities.rs | 39 +--- ethexe/malachite/core/src/lib.rs | 2 +- ethexe/malachite/core/src/store.rs | 31 --- .../malachite/core/tests/multi_validators.rs | 14 +- ethexe/malachite/service/src/externalities.rs | 132 ++++-------- 6 files changed, 59 insertions(+), 354 deletions(-) diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index 65b0a5acd61..582faeeaab4 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -31,7 +31,7 @@ use crate::{ codec::{decode_value, encode_value}, context::{Height, MalachiteCtx, ProposalPart, ValueId}, - externalities::{CallbackOrigin, Externalities}, + externalities::Externalities, state::State, store::BlockEntry, streaming::ProposalParts, @@ -350,8 +350,7 @@ where self.state.store.store_undecided_proposal(&proposed)?; let block = Block::decode(&mut &proposed.value.block_bytes[..]) .map_err(|e| anyhow!("decoding Block from pending proposal: {e}"))?; - self.record_assembled_block(block, CallbackOrigin::Live) - .await + self.record_assembled_block(block).await }; if let Err(e) = promote.await { error!(?e, "rejecting invalid pending proposal"); @@ -401,8 +400,7 @@ where // Hook process_mb_proposal at proposal-assembly time on the // proposer side. cascade_save guarantees ancestor-first // ordering against the application. - self.record_assembled_block(block, CallbackOrigin::Live) - .await?; + self.record_assembled_block(block).await?; Ok(locally) } @@ -450,8 +448,7 @@ where self.state.store.store_undecided_proposal(&proposed)?; let block = Block::decode(&mut &proposed.value.block_bytes[..]) .map_err(|e| anyhow!("decoding Block from received proposal: {e}"))?; - self.record_assembled_block(block, CallbackOrigin::Live) - .await?; + self.record_assembled_block(block).await?; Ok(Some(proposed)) } } @@ -474,7 +471,7 @@ where .state .commit(certificate.clone()) .map_err(FinalizationError::NonFatal)?; - self.ingest_finalized(certificate, block_bytes, CallbackOrigin::Live) + self.ingest_finalized(certificate, block_bytes) .await .context("ingest finalized") .map_err(FinalizationError::Fatal) @@ -504,8 +501,7 @@ where // missing ancestor anyway — see `Store::save_chain`). let block = Block::decode(&mut &proposed.value.block_bytes[..]) .map_err(|e| anyhow!("decoding Block from synced value: {e}"))?; - self.record_assembled_block(block, CallbackOrigin::ValueSyncReplay) - .await?; + self.record_assembled_block(block).await?; } Ok(parsed) } @@ -639,9 +635,8 @@ where /// dedup is idempotent and `cascade_save` skips already-saved /// entries, so the application's `process_mb_proposal` runs at /// most once per `block_hash`. - async fn record_assembled_block(&self, block: Block, origin: CallbackOrigin) -> Result<()> { + async fn record_assembled_block(&self, block: Block) -> Result<()> { let block_hash = block.hash(); - self.state.store.set_mb_origin_once(block_hash, origin)?; self.state.store.insert_block(BlockEntry { block_hash, parent_hash: block.parent_hash, @@ -656,12 +651,7 @@ where .store .cascade_save(vec![block_hash], |hash, blk| { let ext = Arc::clone(&self.externalities); - let store = self.state.store.clone(); - let fallback = origin; - async move { - let cb_origin = store.mb_origin(hash)?.unwrap_or(fallback); - ext.process_mb_proposal(hash, blk, cb_origin).await - } + async move { ext.process_mb_proposal(hash, blk).await } }) .await } @@ -677,12 +667,7 @@ where /// [`Store::cascade_finalize`] silently no-ops on an unsaved /// ancestor (the `finalize_chain` walk returns `None`), and the /// `errors_tx` channel surfaces the contract breach upstream. - async fn ingest_finalized( - &self, - cert: EngineCert, - block_bytes: Vec, - fallback_origin: CallbackOrigin, - ) -> Result<()> { + async fn ingest_finalized(&self, cert: EngineCert, block_bytes: Vec) -> Result<()> { let block = Block::decode(&mut &block_bytes[..]) .map_err(|e| anyhow!("decoding Block at finalize: {e}"))?; let block_hash = block.hash(); @@ -730,11 +715,7 @@ where .store .cascade_finalize(vec![block_hash], |hash, cert| { let ext = Arc::clone(&self.externalities); - let store = self.state.store.clone(); - async move { - let cb_origin = store.mb_origin(hash)?.unwrap_or(fallback_origin); - ext.process_mb_finalized(hash, cert, cb_origin).await - } + async move { ext.process_mb_finalized(hash, cert).await } }) .await?; Ok(()) @@ -780,7 +761,6 @@ fn compute_value_id_from_parts(parts: &ProposalParts) -> ValueId { mod tests { use super::*; use crate::{ - codec::encode_value, context::{ProposalData, ProposalInit, Validator, ValidatorSet, Value}, signing::{MalachiteSigner, libp2p_peer_id, private_key_from_bytes}, state::SharedValidatorSet, @@ -792,11 +772,7 @@ mod tests { ConsensusRequest, NetworkRequest, app::{events::TxEvent, streaming::StreamId}, }; - use malachitebft_core_types::Value as MalachiteBftValue; - use std::{ - sync::{Arc, Mutex}, - time::Duration, - }; + use std::{sync::Arc, time::Duration}; use tempfile::TempDir; use tokio::sync::mpsc; @@ -808,15 +784,10 @@ mod tests { #[async_trait] impl Externalities for NoopExt { - async fn process_mb_proposal(&self, _: H256, _: Block, _: CallbackOrigin) -> Result<()> { + async fn process_mb_proposal(&self, _: H256, _: Block) -> Result<()> { Ok(()) } - async fn process_mb_finalized( - &self, - _: H256, - _: CommitCertificate, - _: CallbackOrigin, - ) -> Result<()> { + async fn process_mb_finalized(&self, _: H256, _: CommitCertificate) -> Result<()> { Ok(()) } async fn build_block_above(&self, _: H256) -> Result { @@ -827,65 +798,6 @@ mod tests { } } - #[derive(Clone, Default)] - struct OriginRecordingExt { - proposal_origins: Arc>>, - finalized_origins: Arc>>, - } - - impl OriginRecordingExt { - fn proposal_origins(&self) -> Vec { - self.proposal_origins - .lock() - .expect("proposal_origins poisoned") - .clone() - } - - fn finalized_origins(&self) -> Vec { - self.finalized_origins - .lock() - .expect("finalized_origins poisoned") - .clone() - } - } - - #[async_trait] - impl Externalities for OriginRecordingExt { - async fn process_mb_proposal( - &self, - _: H256, - _: Block, - origin: CallbackOrigin, - ) -> Result<()> { - self.proposal_origins - .lock() - .expect("proposal_origins poisoned") - .push(origin); - Ok(()) - } - - async fn process_mb_finalized( - &self, - _: H256, - _: CommitCertificate, - origin: CallbackOrigin, - ) -> Result<()> { - self.finalized_origins - .lock() - .expect("finalized_origins poisoned") - .push(origin); - Ok(()) - } - - async fn build_block_above(&self, _: H256) -> Result { - Ok(test_payload()) - } - - async fn validate_block_above(&self, _: H256, _: BlockPayload) -> Result { - Ok(true) - } - } - fn test_signer(byte: u8) -> MalachiteSigner { let mut k = [0u8; 32]; k[31] = byte; @@ -928,76 +840,6 @@ mod tests { ] } - /// Sync-path callbacks must be tagged as - /// `CallbackOrigin::ValueSyncReplay` all the way down to - /// application callbacks. - #[tokio::test] - async fn process_synced_value_forwards_value_sync_replay_origin() { - let ext = OriginRecordingExt::default(); - let handler_ext = ext.clone(); - let (mut handler, _dir, proposer) = make_handler_with(1, handler_ext); - let block = Block::new(H256::zero(), 1, test_payload()); - let block_bytes = block.encode(); - let value_bytes = encode_value(&crate::context::Value::new(block_bytes.clone())); - - let parsed = handler - .process_synced_value(Height::new(1), Round::new(0), proposer, value_bytes) - .await - .expect("process_synced_value should decode and persist the block"); - assert!(parsed.is_some()); - - assert_eq!( - ext.proposal_origins(), - vec![CallbackOrigin::ValueSyncReplay] - ); - - let value_id = crate::context::Value::new(block_bytes).id(); - let cert = malachitebft_core_types::CommitCertificate { - height: Height::new(1), - round: Round::new(0), - value_id, - commit_signatures: Vec::new(), - }; - assert!( - handler.process_finalized(cert).await.is_ok(), - "synced block must be finalized" - ); - assert_eq!( - ext.finalized_origins(), - vec![CallbackOrigin::ValueSyncReplay] - ); - } - - /// Peer/engine proposal callbacks are marked as live traffic and - /// are not mistaken for sync replay callbacks. - #[tokio::test] - async fn process_received_proposal_part_marks_live_origin() { - let ext = OriginRecordingExt::default(); - let handler_ext = ext.clone(); - let (mut handler, _dir, address) = make_handler_with(1, handler_ext); - let block = Block::new(H256::zero(), 1, test_payload()); - let stream = complete_stream(address, 1, &block.encode()); - - run_stream(&mut handler, test_peer(2), stream) - .await - .expect("process_received_proposal_part should assemble a complete proposal"); - - assert_eq!(ext.proposal_origins(), vec![CallbackOrigin::Live]); - - let value_id = crate::context::Value::new(block.encode()).id(); - let cert = malachitebft_core_types::CommitCertificate { - height: Height::new(1), - round: Round::new(0), - value_id, - commit_signatures: Vec::new(), - }; - assert!( - handler.process_finalized(cert).await.is_ok(), - "proposal should finalize" - ); - assert_eq!(ext.finalized_origins(), vec![CallbackOrigin::Live]); - } - /// Build an [`AppMsgHandler`] with the given `current_height` and /// a fresh on-disk store. Channels are dummy senders/receivers /// — `process_received_proposal_part` never touches them on the @@ -1125,15 +967,10 @@ mod tests { #[async_trait] impl Externalities for FailingFinalizeExt { - async fn process_mb_proposal(&self, _: H256, _: Block, _: CallbackOrigin) -> Result<()> { + async fn process_mb_proposal(&self, _: H256, _: Block) -> Result<()> { Ok(()) } - async fn process_mb_finalized( - &self, - _: H256, - _: CommitCertificate, - _: CallbackOrigin, - ) -> Result<()> { + async fn process_mb_finalized(&self, _: H256, _: CommitCertificate) -> Result<()> { Err(anyhow!("application: finalize-side store write failed")) } async fn build_block_above(&self, _: H256) -> Result { @@ -1226,7 +1063,7 @@ mod tests { // `all_ancestors_saved` returns false and the debug-build // assertion fires before the cascade). handler - .record_assembled_block(block, CallbackOrigin::Live) + .record_assembled_block(block) .await .expect("record_assembled_block must succeed under NoopFinalize proposal-side"); diff --git a/ethexe/malachite/core/src/externalities.rs b/ethexe/malachite/core/src/externalities.rs index e5408e174a6..8b7f60b9a8d 100644 --- a/ethexe/malachite/core/src/externalities.rs +++ b/ethexe/malachite/core/src/externalities.rs @@ -9,21 +9,6 @@ use crate::{ }; use anyhow::Result; use async_trait::async_trait; -use parity_scale_codec::{Decode, Encode}; -use serde::{Deserialize, Serialize}; - -/// Indicates where a proposed/finalized MB callback originated. -/// `CallbackOrigin::ValueSyncReplay` identifies callbacks produced by -/// Malachite replay (ValueSync decided-values). It is *provenance only*: -/// applications must decide how to act on this origin, and the default -/// execution path is to treat it like a normal callback unless a higher -/// layer explicitly opts into replay filtering. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Decode, Encode, Deserialize, Serialize)] -pub enum CallbackOrigin { - #[default] - Live, - ValueSyncReplay, -} /// Application-side callbacks the consensus service requires. /// @@ -63,34 +48,14 @@ pub trait Externalities: Send + Sync + 'static { /// per `mb_hash` over the lifetime of an application instance, /// at proposal-assembly time, after every ancestor's /// `process_mb_proposal` has already returned `Ok`. - /// `origin` indicates whether the callback came from local/live - /// operation (`Live`) or Malachite replay (`ValueSyncReplay`). - /// `ValueSyncReplay` is provenance only; downstream implementations - /// may apply additional policy (for example, replay suppression when - /// an execution snapshot boundary is explicitly enabled). - async fn process_mb_proposal( - &self, - mb_hash: H256, - block: Block, - origin: CallbackOrigin, - ) -> Result<()>; + async fn process_mb_proposal(&self, mb_hash: H256, block: Block) -> Result<()>; /// Mark `mb_hash` as finalized and durable. /// /// `cert` is the BFT commit certificate for the height of /// `mb_hash`. The application typically forwards `cert` to /// downstream layers (on-chain commits, light clients, etc.). - /// `origin` indicates whether the callback came from local/live - /// operation (`Live`) or Malachite replay (`ValueSyncReplay`). - /// `ValueSyncReplay` is provenance only; downstream implementations - /// may apply additional policy (for example, replay suppression when - /// an execution snapshot boundary is explicitly enabled). - async fn process_mb_finalized( - &self, - mb_hash: H256, - cert: CommitCertificate, - origin: CallbackOrigin, - ) -> Result<()>; + async fn process_mb_finalized(&self, mb_hash: H256, cert: CommitCertificate) -> Result<()>; /// Build a fresh block payload whose parent has hash /// `parent_mb_hash`. Called only when this node has been elected diff --git a/ethexe/malachite/core/src/lib.rs b/ethexe/malachite/core/src/lib.rs index da0c51850fd..19a8f9bc882 100644 --- a/ethexe/malachite/core/src/lib.rs +++ b/ethexe/malachite/core/src/lib.rs @@ -93,7 +93,7 @@ pub use crate::{ Environment as MalachiteConfigEnvironment, MalachiteConfig, Multiaddr, NodeRole, ValidatorEntry, }, - externalities::{CallbackOrigin, Externalities}, + externalities::Externalities, service::{MService, MalachiteService}, signing::{ MalachiteSigner, PrivateKey, PublicKey, Signature, derive_libp2p_secret, diff --git a/ethexe/malachite/core/src/store.rs b/ethexe/malachite/core/src/store.rs index 626a7c46c99..d9fbbe0a8fb 100644 --- a/ethexe/malachite/core/src/store.rs +++ b/ethexe/malachite/core/src/store.rs @@ -16,7 +16,6 @@ //! - `0x05` `(height,round,value_id)` → engine undecided proposal //! - `0x06` `(height,round,value_id)` → buffered proposal parts //! - `0x07` `height_be[8]` → engine-side `CommitCertificate` -//! - `0x08` `mb_hash[32]` → callback origin for per-block processing //! //! Children of the genesis (parent_hash == [`H256::zero`]) live under //! the bare-zero parent key — same shape as any other parent. @@ -38,7 +37,6 @@ use rocksdb::{DB, Options, WriteBatch}; use crate::{ BlockPayload, context::Height, - externalities::CallbackOrigin, types::{Block, CommitCertificate, H256}, }; @@ -50,7 +48,6 @@ mod prefix { pub const UNDECIDED: u8 = 0x05; pub const PENDING_PARTS: u8 = 0x06; pub const ENGINE_CERT: u8 = 0x07; - pub const MB_ORIGIN: u8 = 0x08; } const META_LATEST_FINALIZED: &[u8] = b"latest_finalized"; @@ -132,13 +129,6 @@ impl Store { k } - fn key_mb_origin(hash: H256) -> [u8; 33] { - let mut k = [0u8; 33]; - k[0] = prefix::MB_ORIGIN; - k[1..33].copy_from_slice(hash.as_bytes()); - k - } - fn decode_one(bytes: &[u8], what: &'static str) -> Result { T::decode(&mut &bytes[..]).with_context(|| format!("decoding {what}")) } @@ -192,27 +182,6 @@ impl Store { } } - /// Persist callback origin for this MB once. Already-present entries - /// are preserved so the first observer of this MB wins. - pub fn set_mb_origin_once(&self, block_hash: H256, origin: CallbackOrigin) -> Result<()> { - let key = Self::key_mb_origin(block_hash); - if self.db.get(key)?.is_none() { - self.db.put(key, origin.encode())?; - } - Ok(()) - } - - /// Load the callback origin associated with this MB, if present. - pub fn mb_origin(&self, block_hash: H256) -> Result> { - match self.db.get(Self::key_mb_origin(block_hash))? { - Some(b) => Ok(Some(Self::decode_one::( - &b, - "mb callback origin", - )?)), - None => Ok(None), - } - } - /// Mark the block as `saved`. Idempotent. Errors if the block /// isn't in the store yet — the caller must `insert_block` first. pub fn mark_saved(&self, block_hash: H256) -> Result<()> { diff --git a/ethexe/malachite/core/tests/multi_validators.rs b/ethexe/malachite/core/tests/multi_validators.rs index 52da41942a0..72e6c26843e 100644 --- a/ethexe/malachite/core/tests/multi_validators.rs +++ b/ethexe/malachite/core/tests/multi_validators.rs @@ -103,12 +103,7 @@ impl TestExt { #[async_trait] impl Externalities for TestExt { - async fn process_mb_proposal( - &self, - hash: H256, - block: Block, - _origin: ethexe_malachite_core::CallbackOrigin, - ) -> Result<()> { + async fn process_mb_proposal(&self, hash: H256, block: Block) -> Result<()> { let mut s = self.state.lock().unwrap(); if block.hash() != hash { s.violations @@ -129,12 +124,7 @@ impl Externalities for TestExt { Ok(()) } - async fn process_mb_finalized( - &self, - hash: H256, - cert: CommitCertificate, - _origin: ethexe_malachite_core::CallbackOrigin, - ) -> Result<()> { + async fn process_mb_finalized(&self, hash: H256, cert: CommitCertificate) -> Result<()> { let mut s = self.state.lock().unwrap(); if cert.block_hash != hash { s.violations diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index d3bcf06d99d..e5c9c09ae4c 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -51,9 +51,7 @@ use ethexe_common::{ malachite::{Operation, Operations}, }; use ethexe_db::Database; -use ethexe_malachite_core::{ - Block, BlockPayload, CallbackOrigin, Externalities, MAX_BLOCK_PAYLOAD_BYTES, -}; +use ethexe_malachite_core::{Block, BlockPayload, Externalities, MAX_BLOCK_PAYLOAD_BYTES}; use gprimitives::H256; use parity_scale_codec::{DecodeAll, Encode}; use std::{ @@ -134,12 +132,7 @@ pub(crate) struct PendingEvent { #[async_trait] impl Externalities for EthexeExternalities { - async fn process_mb_proposal( - &self, - mb_hash: H256, - mb: Block, - _origin: CallbackOrigin, - ) -> Result<()> { + async fn process_mb_proposal(&self, mb_hash: H256, mb: Block) -> Result<()> { // Runs on the proposer, participant, and sync paths. Frozen // discriminants mean every historical operation always decodes; an // unknown one can only come from a newer protocol this build doesn't @@ -204,7 +197,6 @@ impl Externalities for EthexeExternalities { &self, mb_hash: H256, cert: ethexe_malachite_core::CommitCertificate, - _origin: CallbackOrigin, ) -> Result<()> { let compact = self.db.mb_compact_block(mb_hash).ok_or_else(|| { anyhow!( @@ -843,8 +835,6 @@ mod tests { db::{BlockMetaStorageRW, OnChainStorageRW}, injected::PurgedTransaction, }; - use ethexe_malachite_core::CallbackOrigin; - fn to_payload(bytes: Vec) -> BlockPayload { BlockPayload::try_from(bytes).expect("test payload within size cap") } @@ -939,9 +929,7 @@ mod tests { let p = payload(None, 1); let block = wrap(p.clone(), 1, H256::zero()); let mb_hash = block.hash(); - ext.process_mb_proposal(mb_hash, block, CallbackOrigin::Live) - .await - .unwrap(); + ext.process_mb_proposal(mb_hash, block).await.unwrap(); let compact = db.mb_compact_block(mb_hash).expect("CompactMb saved"); assert_eq!(compact.height, 1); @@ -979,11 +967,9 @@ mod tests { let p = payload(None, 5); let block = wrap(p.clone(), 1, H256::zero()); let mb_hash = block.hash(); - ext.process_mb_proposal(mb_hash, block, CallbackOrigin::Live) - .await - .unwrap(); + ext.process_mb_proposal(mb_hash, block).await.unwrap(); let _ = rx.recv().await; // BlockProposal - ext.process_mb_finalized(mb_hash, fake_cert(1), CallbackOrigin::Live) + ext.process_mb_finalized(mb_hash, fake_cert(1)) .await .unwrap(); assert_eq!(db.globals().latest_finalized_mb_hash, mb_hash); @@ -1019,12 +1005,9 @@ mod tests { let p = payload(None, i as u8); let block = wrap(p.clone(), i, parent); let mb_hash = block.hash(); + ext_a.process_mb_proposal(mb_hash, block).await.unwrap(); ext_a - .process_mb_proposal(mb_hash, block, CallbackOrigin::Live) - .await - .unwrap(); - ext_a - .process_mb_finalized(mb_hash, fake_cert(i), CallbackOrigin::Live) + .process_mb_finalized(mb_hash, fake_cert(i)) .await .unwrap(); chain.push((mb_hash, p)); @@ -1054,15 +1037,9 @@ mod tests { let p4 = payload(None, 99); let block4 = wrap(p4.clone(), 4, last_pre); let mb4 = block4.hash(); - ext_b - .process_mb_proposal(mb4, block4, CallbackOrigin::Live) - .await - .unwrap(); + ext_b.process_mb_proposal(mb4, block4).await.unwrap(); let _ = rx_b.recv().await; // proposal - ext_b - .process_mb_finalized(mb4, fake_cert(4), CallbackOrigin::Live) - .await - .unwrap(); + ext_b.process_mb_finalized(mb4, fake_cert(4)).await.unwrap(); assert_eq!(db.mb_compact_block(mb4).unwrap().parent, last_pre); assert_eq!(db.globals().latest_finalized_mb_hash, mb4); } @@ -1087,10 +1064,8 @@ mod tests { let height = (i + 1) as u64; let block = wrap(p.clone(), height, parent); let mb_hash = block.hash(); - ext.process_mb_proposal(mb_hash, block, CallbackOrigin::Live) - .await - .unwrap(); - ext.process_mb_finalized(mb_hash, fake_cert(height), CallbackOrigin::Live) + ext.process_mb_proposal(mb_hash, block).await.unwrap(); + ext.process_mb_finalized(mb_hash, fake_cert(height)) .await .unwrap(); chain.push(mb_hash); @@ -1158,11 +1133,7 @@ mod tests { let (ext, _rx) = make_externalities(db.clone()); let block = Block::new(H256::zero(), 1, to_payload(vec![0xff, 0xff, 0xff, 0xff])); let mb_hash = block.hash(); - assert!( - ext.process_mb_proposal(mb_hash, block, CallbackOrigin::Live) - .await - .is_err() - ); + assert!(ext.process_mb_proposal(mb_hash, block).await.is_err()); } #[tokio::test] @@ -1334,9 +1305,7 @@ mod tests { ]); let block = Block::new(H256::zero(), 1, to_payload(payload.encode())); let mb_hash = block.hash(); - ext.process_mb_proposal(mb_hash, block, CallbackOrigin::Live) - .await - .unwrap(); + ext.process_mb_proposal(mb_hash, block).await.unwrap(); // Drain the BlockProposal event the save emits. let _ = event_rx.recv().await; ext.process_mb_finalized( @@ -1346,7 +1315,6 @@ mod tests { block_hash: mb_hash, signatures: vec![], }, - CallbackOrigin::Live, ) .await .unwrap(); @@ -1381,7 +1349,7 @@ mod tests { let replay_block = wrap(replay_payload, 9, H256::zero()); let replay_hash = replay_block.hash(); - ext.process_mb_proposal(replay_hash, replay_block, CallbackOrigin::ValueSyncReplay) + ext.process_mb_proposal(replay_hash, replay_block) .await .unwrap(); match rx.recv().await.expect("proposal event").unwrap() { @@ -1392,7 +1360,7 @@ mod tests { other => panic!("expected BlockProposal, got {other:?}"), } - ext.process_mb_finalized(replay_hash, fake_cert(9), CallbackOrigin::ValueSyncReplay) + ext.process_mb_finalized(replay_hash, fake_cert(9)) .await .unwrap(); match rx.recv().await.expect("finalization event").unwrap() { @@ -1423,13 +1391,9 @@ mod tests { let boundary_hash = boundary_block.hash(); ext.enable_fast_sync_replay_filter(boundary_hash, boundary_eb); - ext.process_mb_proposal( - boundary_hash, - boundary_block, - CallbackOrigin::ValueSyncReplay, - ) - .await - .unwrap(); + ext.process_mb_proposal(boundary_hash, boundary_block) + .await + .unwrap(); assert!(rx.try_recv().is_err(), "boundary proposal is suppressed"); assert!( ext.pending_events @@ -1440,13 +1404,9 @@ mod tests { assert_eq!(db.mb_meta(boundary_hash).last_advanced_eb, boundary_eb); assert!(db.mb_compact_block(boundary_hash).is_some()); - ext.process_mb_finalized( - boundary_hash, - fake_cert(10), - CallbackOrigin::ValueSyncReplay, - ) - .await - .unwrap(); + ext.process_mb_finalized(boundary_hash, fake_cert(10)) + .await + .unwrap(); assert!( rx.try_recv().is_err(), "boundary finalization is suppressed" @@ -1464,7 +1424,7 @@ mod tests { let post_block = wrap(post_payload, 11, boundary_hash); let post_hash = post_block.hash(); - ext.process_mb_proposal(post_hash, post_block, CallbackOrigin::ValueSyncReplay) + ext.process_mb_proposal(post_hash, post_block) .await .unwrap(); match rx.recv().await.expect("proposal event").unwrap() { @@ -1475,7 +1435,7 @@ mod tests { other => panic!("expected BlockProposal, got {other:?}"), } - ext.process_mb_finalized(post_hash, fake_cert(11), CallbackOrigin::ValueSyncReplay) + ext.process_mb_finalized(post_hash, fake_cert(11)) .await .unwrap(); match rx.recv().await.expect("finalization event").unwrap() { @@ -1514,13 +1474,9 @@ mod tests { }, ); - ext.process_mb_finalized( - boundary_hash, - fake_cert(10), - CallbackOrigin::ValueSyncReplay, - ) - .await - .unwrap(); + ext.process_mb_finalized(boundary_hash, fake_cert(10)) + .await + .unwrap(); assert!( rx.try_recv().is_err(), "boundary finalization before proposal is suppressed" @@ -1537,32 +1493,20 @@ mod tests { db.mutate_block_meta(retained_eb, |meta| meta.prepared = true); let retained_block = wrap(payload(Some(retained_eb), 2), 11, boundary_hash); let retained_hash = retained_block.hash(); - ext.process_mb_proposal( - retained_hash, - retained_block, - CallbackOrigin::ValueSyncReplay, - ) - .await - .unwrap(); + ext.process_mb_proposal(retained_hash, retained_block) + .await + .unwrap(); assert!( rx.try_recv().is_err(), "non-boundary proposal is still suppressed while filter is retained" ); - ext.process_mb_proposal( - boundary_hash, - boundary_block, - CallbackOrigin::ValueSyncReplay, - ) - .await - .unwrap(); - ext.process_mb_finalized( - boundary_hash, - fake_cert(10), - CallbackOrigin::ValueSyncReplay, - ) - .await - .unwrap(); + ext.process_mb_proposal(boundary_hash, boundary_block) + .await + .unwrap(); + ext.process_mb_finalized(boundary_hash, fake_cert(10)) + .await + .unwrap(); assert!( ext.fast_sync_replay_filter .read() @@ -1575,7 +1519,7 @@ mod tests { db.mutate_block_meta(post_eb, |meta| meta.prepared = true); let post_block = wrap(payload(Some(post_eb), 3), 12, boundary_hash); let post_hash = post_block.hash(); - ext.process_mb_proposal(post_hash, post_block, CallbackOrigin::ValueSyncReplay) + ext.process_mb_proposal(post_hash, post_block) .await .unwrap(); match rx.recv().await.expect("proposal event").unwrap() { @@ -1599,7 +1543,7 @@ mod tests { ext.enable_fast_sync_replay_filter(mb_hash, expected_eb); let err = ext - .process_mb_proposal(mb_hash, block, CallbackOrigin::ValueSyncReplay) + .process_mb_proposal(mb_hash, block) .await .expect_err("boundary EB mismatch must fail"); assert!( @@ -1636,7 +1580,7 @@ mod tests { ext.enable_fast_sync_replay_filter(mb_hash, expected_eb); let err = ext - .process_mb_proposal(mb_hash, block, CallbackOrigin::ValueSyncReplay) + .process_mb_proposal(mb_hash, block) .await .expect_err("zero boundary EB mismatch must fail"); assert!( From 4a0bc2eb7d58eaf199133bc64d6f81048bd34fd3 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 18:02:48 +0300 Subject: [PATCH 42/72] Clean up fast-sync replay filter tests --- ethexe/malachite/service/src/externalities.rs | 90 ++++--------------- 1 file changed, 19 insertions(+), 71 deletions(-) diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index e5c9c09ae4c..204b13b3250 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -62,15 +62,15 @@ use tokio::sync::{Notify, mpsc}; use tracing::{error, info, warn}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct FastSyncReplayBoundary { +pub(crate) struct FastSyncReplayTarget { pub mb_hash: H256, pub eb_hash: H256, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct FastSyncReplayFilter { - pub boundary: FastSyncReplayBoundary, - pub boundary_proposal_seen: bool, + pub target: FastSyncReplayTarget, + pub target_proposal_seen: bool, } /// Inputs the externalities need to satisfy the [`ethexe_malachite_core::Externalities`] @@ -642,8 +642,8 @@ impl EthexeExternalities { .fast_sync_replay_filter .write() .expect("fast_sync_replay_filter poisoned") = Some(FastSyncReplayFilter { - boundary: FastSyncReplayBoundary { mb_hash, eb_hash }, - boundary_proposal_seen: false, + target: FastSyncReplayTarget { mb_hash, eb_hash }, + target_proposal_seen: false, }); } @@ -661,13 +661,13 @@ impl EthexeExternalities { return Ok(false); }; - let boundary = filter.boundary; - let matches_boundary = mb_hash == boundary.mb_hash && last_advanced_eb == boundary.eb_hash; + let target = filter.target; + let matches_target = mb_hash == target.mb_hash && last_advanced_eb == target.eb_hash; - if mb_hash == boundary.mb_hash && !matches_boundary { + if mb_hash == target.mb_hash && !matches_target { let message = format!( - "fast-sync replay boundary mismatch: MB {mb_hash} advanced {last_advanced_eb}, expected {}", - boundary.eb_hash, + "fast-sync replay target mismatch: MB {mb_hash} advanced {last_advanced_eb}, expected {}", + target.eb_hash, ); *guard = None; drop(guard); @@ -676,11 +676,11 @@ impl EthexeExternalities { return Err(anyhow!(message)); } - if matches_boundary { + if matches_target { guard .as_mut() .expect("fast-sync replay filter checked above") - .boundary_proposal_seen = true; + .target_proposal_seen = true; } Ok(true) @@ -696,7 +696,7 @@ impl EthexeExternalities { return false; }; - if mb_hash == filter.boundary.mb_hash && filter.boundary_proposal_seen { + if mb_hash == filter.target.mb_hash && filter.target_proposal_seen { *guard = None; } @@ -1330,54 +1330,6 @@ mod tests { assert!(seen_hashes.contains(&tx_b.data().to_hash())); } - /// Nonzero execution globals are not enough to suppress ValueSync replay. - /// Suppression must be explicitly enabled after fast-sync capture. - #[tokio::test] - async fn value_sync_replay_without_enabled_boundary_processes_normally() { - let db = Database::memory(); - let (ext, mut rx) = make_externalities(db.clone()); - - let boundary_payload = payload(None, 1); - let boundary_block = wrap(boundary_payload, 10, H256::zero()); - let boundary_hash = boundary_block.hash(); - db.globals_mutate(|g| { - g.latest_finalized_mb_hash = boundary_hash; - g.latest_computed_mb_hash = boundary_hash; - }); - - let replay_payload = payload(None, 2); - let replay_block = wrap(replay_payload, 9, H256::zero()); - let replay_hash = replay_block.hash(); - - ext.process_mb_proposal(replay_hash, replay_block) - .await - .unwrap(); - match rx.recv().await.expect("proposal event").unwrap() { - MalachiteEvent::BlockProposal { height, mb_hash } => { - assert_eq!(height, 9); - assert_eq!(mb_hash, replay_hash); - } - other => panic!("expected BlockProposal, got {other:?}"), - } - - ext.process_mb_finalized(replay_hash, fake_cert(9)) - .await - .unwrap(); - match rx.recv().await.expect("finalization event").unwrap() { - MalachiteEvent::BlockFinalized { - cert, - height, - mb_hash, - } => { - assert_eq!(height, 9); - assert_eq!(cert.height, 9); - assert_eq!(mb_hash, replay_hash); - } - other => panic!("expected BlockFinalized, got {other:?}"), - } - assert_eq!(db.globals().latest_finalized_mb_hash, replay_hash); - } - #[tokio::test] async fn fast_sync_filter_suppresses_until_boundary_finalization() { use ethexe_common::db::MbStorageRO; @@ -1545,16 +1497,14 @@ mod tests { let err = ext .process_mb_proposal(mb_hash, block) .await - .expect_err("boundary EB mismatch must fail"); + .expect_err("target EB mismatch must fail"); assert!( - err.to_string() - .contains("fast-sync replay boundary mismatch"), + err.to_string().contains("fast-sync replay target mismatch"), "unexpected error: {err:#}" ); match rx.try_recv().expect("mismatch event") { Err(err) => assert!( - err.to_string() - .contains("fast-sync replay boundary mismatch"), + err.to_string().contains("fast-sync replay target mismatch"), "unexpected event error: {err:#}" ), Ok(event) => panic!("expected mismatch error event, got {event:?}"), @@ -1582,16 +1532,14 @@ mod tests { let err = ext .process_mb_proposal(mb_hash, block) .await - .expect_err("zero boundary EB mismatch must fail"); + .expect_err("zero target EB mismatch must fail"); assert!( - err.to_string() - .contains("fast-sync replay boundary mismatch"), + err.to_string().contains("fast-sync replay target mismatch"), "unexpected error: {err:#}" ); match rx.try_recv().expect("mismatch event") { Err(err) => assert!( - err.to_string() - .contains("fast-sync replay boundary mismatch"), + err.to_string().contains("fast-sync replay target mismatch"), "unexpected event error: {err:#}" ), Ok(event) => panic!("expected mismatch error event, got {event:?}"), From a2146e1a551fbe59afa77203e3592ba66bf98f9c Mon Sep 17 00:00:00 2001 From: Gregory Sobol Date: Thu, 11 Jun 2026 17:14:02 +0200 Subject: [PATCH 43/72] more errors handling --- Cargo.lock | 1 + ethexe/malachite/core/Cargo.toml | 1 + ethexe/malachite/core/src/app.rs | 126 +++++++++++++++++-------------- 3 files changed, 73 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index abc97d909a4..8aec710e813 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5712,6 +5712,7 @@ dependencies = [ "arc-malachitebft-test", "async-trait", "bytes", + "derive_more 2.1.1", "ethexe-common", "futures", "gear-core", diff --git a/ethexe/malachite/core/Cargo.toml b/ethexe/malachite/core/Cargo.toml index 3d419bd8b64..cac2ed1a0d8 100644 --- a/ethexe/malachite/core/Cargo.toml +++ b/ethexe/malachite/core/Cargo.toml @@ -39,6 +39,7 @@ tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync tracing.workspace = true libp2p-identity.workspace = true rocksdb.workspace = true +derive_more.workspace = true [dev-dependencies] advisory-lock.workspace = true diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index 15abe12e631..691c0d60098 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -56,7 +56,7 @@ use malachitebft_app_channel::{ use malachitebft_core_types::Height as _; use parity_scale_codec::{Decode, Encode}; use std::{ops::RangeInclusive, sync::Arc}; -use tracing::{error, info, trace, warn}; +use tracing::{debug, error, info, trace, warn}; /// Max allowed distance into the future for pending proposal parts. const FUTURE_HEIGHT_WINDOW: u64 = 4; @@ -89,6 +89,14 @@ enum FinalizationError { NonFatal(anyhow::Error), } +#[derive(derive_more::From, derive_more::Display)] +enum ProcessGetValueError { + #[display("building block timeout elapsed: {_0}")] + TimeoutElapsed(tokio::time::error::Elapsed), + #[display(" {_0}")] + Any(anyhow::Error), +} + /// Owns the channel-app event-loop state and dispatches each /// [`AppMsg`] variant to its matching `process_*` method. The dispatch /// holds the engine reply channel — `process_*` only produces the @@ -108,7 +116,10 @@ where let Some(msg) = self.channels.consensus.recv().await else { return Err(anyhow!("consensus channel closed")); }; - self.handle_app_msg(msg).await?; + + if let Err(err) = self.handle_app_msg(msg).await { + error!("error handling AppMsg: {err}"); + } } } @@ -116,9 +127,9 @@ where match msg { // ConsensusReady AppMsg::ConsensusReady { reply } => { - if reply.send(self.process_consensus_ready()).is_err() { - error!("ConsensusReady: failed to send reply"); - } + reply + .send(self.process_consensus_ready()) + .map_err(|e| anyhow!("failed to send ConsensusReady reply: {e:?}"))?; } // StartedRound @@ -137,9 +148,9 @@ where error!(?e, %height, %round, "StartedRound: process failed"); Vec::new() }); - if reply_value.send(proposals).is_err() { - error!("StartedRound: failed to send proposals reply"); - } + reply_value + .send(proposals) + .map_err(|e| anyhow!("failed to send StartedRound reply: {e:?}"))?; } // GetValue (we are proposer) @@ -152,35 +163,44 @@ where info!(%height, %round, "GetValue"); match self.process_get_value(height, round).await { Ok(proposal) => { - if reply.send(proposal.clone()).is_err() { - error!("GetValue: failed to send proposal reply"); - } + reply + .send(proposal.clone()) + .map_err(|e| anyhow!("failed to send GetValue reply: {e:?}"))?; + for stream_message in self.state.stream_proposal(proposal, Round::Nil) { - self.channels + if let Err(err) = self + .channels .network .send(NetworkMsg::PublishProposalPart(stream_message)) - .await?; + .await + { + error!("failed to send PublishProposalPart: {err}"); + } } } - Err(e) => { - // No usable default for `LocallyProposedValue` — - // dropping the reply sender lets the engine time - // out the propose step and advance the round. - error!(?e, %height, %round, "GetValue: process failed — skipping reply"); + + // No usable default for `LocallyProposedValue` — + // dropping the reply sender lets the engine time + // out the propose step and advance the round. + Err(ProcessGetValueError::TimeoutElapsed(e)) => { + warn!(%height, %round, reason = %e, "unable to produce proposal"); + } + Err(ProcessGetValueError::Any(e)) => { + return Err(anyhow!("errors during proposal production: {e:?}")); } } } // Vote extensions (unused — return defaults). AppMsg::ExtendVote { reply, .. } => { - if reply.send(self.process_extend_vote()).is_err() { - error!("ExtendVote: failed to send reply"); - } + reply + .send(self.process_extend_vote()) + .map_err(|e| anyhow!("failed to send ExtendVote reply: {e:?}"))?; } AppMsg::VerifyVoteExtension { reply, .. } => { - if reply.send(self.process_verify_vote_extension()).is_err() { - error!("VerifyVoteExtension: failed to send reply"); - } + reply + .send(self.process_verify_vote_extension()) + .map_err(|e| anyhow!("failed to send VerifyVoteExtension reply: {e:?}"))?; } // ReceivedProposalPart (we are not proposer) @@ -197,9 +217,9 @@ where error!("ReceivedProposalPart: process failed, {e}"); None }); - if reply.send(value).is_err() { - error!("ReceivedProposalPart: failed to send reply"); - } + reply + .send(value) + .map_err(|e| anyhow!("failed to send ReceivedProposalPart reply: {e:?}"))?; } // Decided (info only — Finalized fires next). @@ -250,9 +270,9 @@ where return Err(anyhow!("Fatal error during finalization: {e:?}")); } }; - if reply.send(next).is_err() { - error!("Finalized: failed to send Next reply"); - } + reply + .send(next) + .map_err(|e| anyhow!("failed to send Next reply: {e:?}"))?; } // Sync path @@ -268,32 +288,32 @@ where .process_synced_value(height, round, proposer, value_bytes) .await .unwrap_or_else(|e| { - error!(?e, %height, %round, "ProcessSyncedValue: process failed"); + error!(?e, %height, %round, "process synced value failed"); None }); - if reply.send(value).is_err() { - error!("ProcessSyncedValue: failed to send reply"); - } + reply + .send(value) + .map_err(|e| anyhow!("failed to send ProcessSyncedValue reply: {e:?}"))? } AppMsg::GetDecidedValues { range, reply } => { let values = self.process_get_decided_values(range).unwrap_or_else(|e| { - error!(?e, "GetDecidedValues: process failed"); + error!(?e, "process get decided values failed"); Vec::new() }); - if reply.send(values).is_err() { - error!("GetDecidedValues: failed to send reply"); - } + reply + .send(values) + .map_err(|e| anyhow!("failed to send GetDecidedValues reply: {e:?}"))?; } AppMsg::GetHistoryMinHeight { reply } => { let h = self.process_get_history_min_height().unwrap_or_else(|e| { - error!(?e, "GetHistoryMinHeight: process failed"); + error!(?e, "process get history min height failed"); Height::default() }); - if reply.send(h).is_err() { - error!("GetHistoryMinHeight: failed to send reply"); - } + reply + .send(h) + .map_err(|e| anyhow!("failed to send GetHistoryMinHeight reply: {e:?}"))?; } AppMsg::RestreamProposal { @@ -302,15 +322,12 @@ where valid_round, address: _, value_id, - } => { - if let Err(e) = self - .process_restream_proposal(height, round, valid_round, value_id) - .await - { - error!(?e, %height, %round, "RestreamProposal: process failed"); - } - } + } => self + .process_restream_proposal(height, round, valid_round, value_id) + .await + .map_err(|e| anyhow!("restream proposal failed: {e:?}"))?, } + Ok(()) } @@ -370,9 +387,9 @@ where &mut self, height: Height, round: Round, - ) -> Result> { + ) -> Result, ProcessGetValueError> { if let Some(p) = self.state.get_previously_built_value(height, round)? { - info!("re-using previously built value"); + debug!("re-using previously built value"); return Ok(p); } @@ -394,9 +411,8 @@ where let build_fut = self.externalities.build_block_above(parent_hash); let payload = tokio::time::timeout(self.state.propose_timeout, build_fut) - .await - .context("block proposal timeout elapsed")? - .context("Proposal block building failed")?; + .await? + .context("build_block_above failed")?; let block = Block::new(parent_hash, height.as_u64(), payload); let block_bytes = block.encode(); let locally = self From b911e0947defcb27856a78d08436f2e7f0342720 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 18:17:01 +0300 Subject: [PATCH 44/72] Skip replay filter for finalized Malachite target --- ethexe/malachite/core/src/service.rs | 14 ++++++++++++-- ethexe/malachite/core/src/store.rs | 19 +++++++++++++++++++ ethexe/malachite/service/src/service.rs | 17 ++++++++++++++++- ethexe/service/src/fast_sync.rs | 4 ++-- 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/ethexe/malachite/core/src/service.rs b/ethexe/malachite/core/src/service.rs index 9f63fb88dca..9818851440d 100644 --- a/ethexe/malachite/core/src/service.rs +++ b/ethexe/malachite/core/src/service.rs @@ -14,7 +14,7 @@ use crate::{ }, state::{SharedValidatorSet, State}, store::Store, - types::Address, + types::{Address, H256}, }; use advisory_lock::{AdvisoryFileLock, FileLockMode}; use anyhow::{Context as _, Result}; @@ -63,6 +63,10 @@ pub struct MalachiteService { /// Shared with the inner app loop; [`Self::update_validators`] /// writes here, the next `Finalized` / `ConsensusReady` reply reads. validator_set: SharedValidatorSet, + /// Persistent app store, shared with the app task. Kept here for + /// startup decisions that need to inspect replay state before the + /// app task is released. + store: Store, _externalities: Arc, } @@ -258,7 +262,7 @@ impl MalachiteService { signer, validator_set.clone(), address, - store, + store.clone(), config.propose_timeout, )?; @@ -282,6 +286,7 @@ impl MalachiteService { app_handle, wal_path, validator_set, + store, _externalities: externalities, }) } @@ -294,6 +299,11 @@ impl MalachiteService { .expect("start receiver has been dropped"); } + /// Whether `block_hash` is already finalized in the persistent app store. + pub fn is_finalized(&self, block_hash: H256) -> Result { + self.store.is_finalized(block_hash) + } + /// Swap the active validator set used at the next height start. /// Malachite's `StartHeight` snapshots the set at the height /// start, so the current height runs to completion with whatever diff --git a/ethexe/malachite/core/src/store.rs b/ethexe/malachite/core/src/store.rs index d9fbbe0a8fb..6cfb5fb51ee 100644 --- a/ethexe/malachite/core/src/store.rs +++ b/ethexe/malachite/core/src/store.rs @@ -330,6 +330,11 @@ impl Store { } } + /// Whether `block_hash` is known and marked finalized. + pub fn is_finalized(&self, block_hash: H256) -> Result { + Ok(self.get_block(block_hash)?.is_some_and(|e| e.finalized)) + } + /// Drive the application's `process_mb_proposal` callback over /// every ancestor that is now ready, starting from each seed. Cascades /// to children: when a block becomes saveable, its descendants are @@ -920,6 +925,20 @@ mod tests { assert!(err.to_string().contains("not saved yet")); } + #[test] + fn is_finalized_tracks_persistent_finalization_state() { + let (_d, store) = open_store(); + + assert!(!store.is_finalized(h(1)).unwrap()); + + store.insert_block(mk_entry(h(1), H256::zero(), 1)).unwrap(); + store.mark_saved(h(1)).unwrap(); + assert!(!store.is_finalized(h(1)).unwrap()); + + store.mark_finalized(h(1), mk_cert(1, h(1))).unwrap(); + assert!(store.is_finalized(h(1)).unwrap()); + } + // --- restart persistence -------------------------------------------- #[test] diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index d81e0eb4ca3..0a85016f8d7 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -198,9 +198,24 @@ impl MalachiteService { } } - pub fn enable_fast_sync_replay_filter(&self, mb_hash: H256, eb_hash: H256) { + pub fn enable_fast_sync_replay_filter(&self, mb_hash: H256, eb_hash: H256) -> Result { + if self + .inner + .as_ref() + .context("Malachite inner service is shut down")? + .is_finalized(mb_hash)? + { + tracing::info!( + mb_hash = %mb_hash, + eb_hash = %eb_hash, + "not enabling fast-sync replay filter for already-finalized MB", + ); + return Ok(false); + } + self.externalities .enable_fast_sync_replay_filter(mb_hash, eb_hash); + Ok(true) } /// Hand an injected transaction to the mempool. The local diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 62e63c8d5d5..43111d4a68d 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -747,10 +747,10 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { if let Some(malachite) = malachite.as_mut() { // `Service::run` performs fast sync before `run_inner().start_app_task()`, // so live Malachite callbacks cannot race this startup replay gate. - malachite.enable_fast_sync_replay_filter( + let _ = malachite.enable_fast_sync_replay_filter( latest_committed_chain.mb_hash, latest_committed_chain.eb_hash, - ); + )?; malachite.receive_new_chain_head(block_data.to_simple()); } From 3253ef4b2f2b7d2ef9b7002b86538e8463a7c052 Mon Sep 17 00:00:00 2001 From: Grisha Sobol Date: Thu, 11 Jun 2026 17:25:16 +0200 Subject: [PATCH 45/72] fix(vara.eth/malachite): address review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - receive_eb_synced: advance latest_synced by height instead of requiring the synced hash to equal the latest observed head — under lagging sync every BlockSynced was dropped and the producer stalled (codex P1) - wait_for_proposable_content: bail instead of panic on missing mempool - quarantine: Option -> Result via ok_or_else instead of with_context - drop redundant double with_validators + clone in service setup - fix "Malachite not local role" log wording - refresh stale `MalachiteConfig` / `MalachiteService::new` / `receive_new_chain_head` mentions in docs and messages - document the single-writer / no-guard-across-await invariant on ChainHead Co-Authored-By: Claude Opus 4.7 --- ethexe/cli/src/params/malachite.rs | 2 +- ethexe/cli/src/params/node.rs | 2 +- ethexe/malachite/service/src/externalities.rs | 4 +- ethexe/malachite/service/src/lib.rs | 16 ++++---- ethexe/malachite/service/src/quarantine.rs | 14 +++---- ethexe/malachite/service/src/service.rs | 38 +++++++++++-------- ethexe/malachite/service/src/starter.rs | 2 +- ethexe/malachite/service/src/types.rs | 3 ++ .../service/tests/restart_resilience.rs | 4 +- ethexe/service/src/lib.rs | 13 +++---- ethexe/service/src/tests/utils/env.rs | 2 +- 11 files changed, 55 insertions(+), 45 deletions(-) diff --git a/ethexe/cli/src/params/malachite.rs b/ethexe/cli/src/params/malachite.rs index 7ec39cb1d25..1e576e62d2f 100644 --- a/ethexe/cli/src/params/malachite.rs +++ b/ethexe/cli/src/params/malachite.rs @@ -72,7 +72,7 @@ pub struct MalachiteParams { impl MalachiteParams { /// Converts CLI/TOML Malachite parameters into a service-ready /// [`MalachiteCliConfig`]. Missing fields fall back to sensible - /// defaults from [`MalachiteConfig`]. + /// defaults from `MalachiteServiceConfig`. pub fn into_config(self) -> Result { let validator_pub_keys = match self.validators_malachite_pub_keys { Some(path) => load_validator_pub_keys_table(&path)?, diff --git a/ethexe/cli/src/params/node.rs b/ethexe/cli/src/params/node.rs index 5af99de6ecf..005c40fa071 100644 --- a/ethexe/cli/src/params/node.rs +++ b/ethexe/cli/src/params/node.rs @@ -95,7 +95,7 @@ pub struct NodeParams { #[serde(rename = "canonical-quarantine")] pub canonical_quarantine: Option, - /// See `MalachiteConfig::post_quarantine_delay`. Default 1. + /// See `MalachiteServiceConfig::post_quarantine_delay`. Default 1. #[arg(long)] #[serde(rename = "post-quarantine-delay")] pub post_quarantine_delay: Option, diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index af5b053f64f..cc8503a9b62 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -504,7 +504,7 @@ impl EthexeExternalities { let chain_head = *self.chain_head.latest_synced.read().await; let Some(mempool) = self.mempool.as_ref() else { - panic!("must never call wait_for_proposable_content when not a validator"); + anyhow::bail!("must never call wait_for_proposable_content when not a validator"); }; let injected_txs = mempool.fetch(chain_head).await; @@ -1536,7 +1536,7 @@ mod tests { /// in `ProcessQueues.gas_allowance` and force every participant to attempt /// an unbounded queue drain. Validator must reject MBs whose /// `gas_allowance` exceeds the protocol cap - /// (`MalachiteConfig::DEFAULT_GAS_ALLOWANCE`). + /// (`MalachiteServiceConfig::DEFAULT_GAS_ALLOWANCE`). #[tokio::test] async fn validate_rejects_gas_allowance_above_default() { let db = Database::memory(); diff --git a/ethexe/malachite/service/src/lib.rs b/ethexe/malachite/service/src/lib.rs index 7181e378c63..eda28d5cb31 100644 --- a/ethexe/malachite/service/src/lib.rs +++ b/ethexe/malachite/service/src/lib.rs @@ -11,8 +11,9 @@ //! [`MalachiteEvent`]s. //! //! `ethexe-service` constructs the service at startup and is the sole consumer of -//! its output `Stream` of [`MalachiteEvent`]; it calls `receive_new_chain_head` -//! on each `ObserverEvent::BlockSynced`. +//! its output `Stream` of [`MalachiteEvent`]; it calls `receive_new_eb` on each +//! `ObserverEvent::Block` and `receive_eb_synced` on each +//! `ObserverEvent::BlockSynced`. //! //! ## Public API //! @@ -27,17 +28,18 @@ //! - [`TxValidity`] (enum) — Validity verdict: `Valid`, `Duplicate`, `Outdated`, … //! //! Driver methods on [`MalachiteService`]: `receive_injected_transaction`, -//! `receive_new_chain_head`, `receive_eb_prepared`, `shutdown`. +//! `receive_new_eb`, `receive_eb_synced`, `receive_eb_prepared`, `shutdown`. //! //! [`TxValidity`] gates inclusion: a producer drops any non-`Valid` tx when //! building an MB, and a validator rejects an entire MB that contains one. //! //! ## Caller Invariants //! -//! - Construct with `MalachiteService::new(config, db, signer, validator_pub_key, -//! mempool)`. A `Some` key starts a `Validator` and must appear in -//! `config.validators`; `None` starts a gossip/sync-only `FullNode`. `new` -//! returns `Err` if `config.validators` is empty or the local key is absent. +//! - Construct with `MalachiteServiceStarter::new(config, validator_config, db, +//! initial_chain_head)` followed by `.start()`. A `Some(validator_config)` +//! starts a `Validator` whose key must appear in `config.validators`; `None` +//! starts a gossip/sync-only `FullNode`. `new` returns `Err` if +//! `config.validators` is empty or the local key is absent from the signer. //! - `BlockProposal` is always emitted before the matching `BlockFinalized` for a //! height; both series are emitted ancestor-first. //! - Tendermint's quorum threshold is `> 2/3` of total voting power across the diff --git a/ethexe/malachite/service/src/quarantine.rs b/ethexe/malachite/service/src/quarantine.rs index a9de9e69508..51048e8b1d5 100644 --- a/ethexe/malachite/service/src/quarantine.rs +++ b/ethexe/malachite/service/src/quarantine.rs @@ -23,7 +23,7 @@ //! - *"quarantine-passed"* means the block has ≥ `canonical_quarantine` //! canonical descendants on top. -use anyhow::{Context as _, Result, anyhow}; +use anyhow::{Result, anyhow}; use ethexe_common::{Acceptance, SimpleBlockData, db::OnChainStorageRO}; use gprimitives::H256; @@ -69,9 +69,9 @@ pub fn is_strict_descendant_of( return Ok(Acceptance::Accepted(())); } - let ancestor = db.block_simple_data(ancestor_hash).with_context(|| { - anyhow!("descendant check: missing header for ancestor {ancestor_hash}") - })?; + let ancestor = db + .block_simple_data(ancestor_hash) + .ok_or_else(|| anyhow!("descendant check: missing header for ancestor {ancestor_hash}"))?; let Some(depth) = candidate.header.height.checked_sub(ancestor.header.height) else { return Ok(Acceptance::Rejected(format!( @@ -93,9 +93,9 @@ pub fn is_strict_descendant_of( ))); } - cursor = db.block_simple_data(parent_hash).with_context(|| { - anyhow!("descendant check: missing header for parent {parent_hash}") - })?; + cursor = db + .block_simple_data(parent_hash) + .ok_or_else(|| anyhow!("descendant check: missing header for parent {parent_hash}"))?; } Ok(Acceptance::Rejected(format!( diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 2458cf43370..8254a4e4ab3 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -79,32 +79,40 @@ impl MalachiteService { /// Handle a fully synced Ethereum block: publish it for the producer's /// quarantine checks, rotate the validator set on era change and GC the mempool. pub async fn receive_eb_synced(&mut self, eb_hash: H256) { - let chain_head = *self.chain_head.latest.read().await; - if chain_head.hash != eb_hash { - tracing::trace!( - chain_head = %chain_head, - synced = %eb_hash, - "synced EB is not the current chain head, ignoring" - ); + let Some(synced) = self.externalities.db.block_simple_data(eb_hash) else { + tracing::error!(synced = %eb_hash, "synced EB header not found in local DB, ignoring"); return; - } + }; - // Publish the synced head for the externalities' quarantine - // checks BEFORE waking the producer, so the woken round - // already sees it. - *self.chain_head.latest_synced.write().await = chain_head; + // Publish the synced head for the externalities' quarantine checks + // BEFORE waking the producer, so the woken round already sees it. + // A synced block may lag the latest observed head — advance whenever + // it is strictly newer than the current synced head. + { + let mut latest_synced = self.chain_head.latest_synced.write().await; + if !latest_synced.hash.is_zero() && synced.header.height <= latest_synced.header.height + { + tracing::trace!( + latest_synced = %*latest_synced, + synced = %synced, + "synced EB is not newer than the current synced head, ignoring" + ); + return; + } + *latest_synced = synced; + } // Notify inner proposer if it waits (see EthexeExternalities::wait_for_proposable_content) self.chain_head.notify.notify_one(); // Rotate before waking the producer so the next round sees the new set. - self.maybe_rotate_validators_for_era(chain_head); + self.maybe_rotate_validators_for_era(synced); if let Some(pool) = self.mempool.as_ref() { - let purged_txs = pool.set_chain_head(chain_head).await; + let purged_txs = pool.set_chain_head(synced).await; if !purged_txs.is_empty() { let event = MalachiteEvent::PurgedTransactions { - eb_hash: chain_head.hash, + eb_hash: synced.hash, transactions: purged_txs, }; if let Err(err) = self.externalities.event_tx.send(Ok(event)) { diff --git a/ethexe/malachite/service/src/starter.rs b/ethexe/malachite/service/src/starter.rs index 45c4ad7698d..d887314b449 100644 --- a/ethexe/malachite/service/src/starter.rs +++ b/ethexe/malachite/service/src/starter.rs @@ -47,7 +47,7 @@ impl MalachiteServiceStarter { .with_context(|| format!("creating Malachite home dir {:?}", config.home_dir))?; if config.validators.is_empty() { - return Err(anyhow!("MalachiteConfig::validators is empty")); + return Err(anyhow!("MalachiteServiceConfig::validators is empty")); } let active_era = db diff --git a/ethexe/malachite/service/src/types.rs b/ethexe/malachite/service/src/types.rs index 6fedb8f05ea..c0e6d6df0c5 100644 --- a/ethexe/malachite/service/src/types.rs +++ b/ethexe/malachite/service/src/types.rs @@ -6,6 +6,9 @@ use tokio::sync::{Notify, RwLock}; /// Ethereum chain-head register shared between [`crate::MalachiteService`] /// (writer) and the externalities (reader). +/// +/// Invariant: only the service event loop writes these fields, and no guard +/// is ever held across an `.await` — keep it that way to stay deadlock-free. pub struct ChainHead { /// Latest observed EB. pub latest: RwLock, diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index cec2f9a2a12..f370c0d78cc 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -115,7 +115,7 @@ fn build_signer(home: &Path) -> (Signer, gsigner::schemes::secp256k1: (signer, pub_key) } -/// Build the MalachiteConfig used by the resilience tests: +/// Build the MalachiteServiceConfig used by the resilience tests: /// quarantine-off (so the producer can advance immediately on each /// new chain head), default listen address, no persistent peers, /// single-validator set so the local node can decide on its own. @@ -262,7 +262,7 @@ async fn single_validator_finalizes_and_recovers_after_restart() { // ---- shutdown -------------------------------------------------- // `shutdown().await` waits for the engine actor + RocksDB store // to drop synchronously — `drop(svc)` alone is fire-and-forget - // and would race the second `MalachiteService::new` against the + // and would race the restarted service against the // RocksDB advisory lock. svc.shutdown().await; // libp2p TCP listener still takes a moment past the actor kill diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 22609592385..cede124539e 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -508,17 +508,17 @@ impl Service { &config.malachite.validator_pub_keys, )?; - let malachite_service_config = MalachiteServiceConfig::from_home_dir(malachite_home) + let malachite_config = MalachiteServiceConfig::from_home_dir(malachite_home) .with_listen_addr(config.malachite.listen_addr) .with_persistent_peers(config.malachite.persistent_peers.clone()) .with_canonical_quarantine(config.node.canonical_quarantine) .with_post_quarantine_delay(config.node.post_quarantine_delay) - .with_validators(malachite_validator_set.clone()); + .with_validators(malachite_validator_set); log::info!( "Malachite listen: {} persistent_peers: {}", - malachite_service_config.listen_addr, - malachite_service_config.persistent_peers.len(), + malachite_config.listen_addr, + malachite_config.persistent_peers.len(), ); let validator_config = @@ -532,10 +532,7 @@ impl Service { .as_ref() .map(|_| "validator") .unwrap_or("full"); - log::info!("Malachite not local role: {role}",); - - let malachite_config = - malachite_service_config.with_validators(malachite_validator_set); + log::info!("Malachite node role: {role}"); MalachiteServiceStarter::new( malachite_config, diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index 84e82942026..fddeb3ef641 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -1036,7 +1036,7 @@ pub struct Node { /// Snapshot of `env.validators` at `new_node` time — drives the /// boot-time filter on `malachite_endpoints` in `start_service`. active_validator_pub_keys: Vec, - /// Port reservation; dropped just before the first `MalachiteService::new`. + /// Port reservation; dropped just before the first malachite service start. malachite_listener: Option, running_service_handle: Option>, From e5eafcc189ea0976e3928d1f4a0aee2cdc9ecfc4 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 18:35:00 +0300 Subject: [PATCH 46/72] Start Malachite app task in service tests --- ethexe/malachite/core/tests/multi_validators.rs | 9 ++++++--- ethexe/malachite/service/tests/restart_resilience.rs | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ethexe/malachite/core/tests/multi_validators.rs b/ethexe/malachite/core/tests/multi_validators.rs index 72e6c26843e..7d1cd4f54ee 100644 --- a/ethexe/malachite/core/tests/multi_validators.rs +++ b/ethexe/malachite/core/tests/multi_validators.rs @@ -306,9 +306,11 @@ async fn start_service( ) -> MalachiteService { let peers = build_multiaddrs_excluding(setups, idx); let config = build_config(setup, setups, peers); - MalachiteService::::new(config, ext) + let mut service = MalachiteService::::new(config, ext) .await - .expect("service starts") + .expect("service starts"); + service.start_app_task(); + service } /// Wait until *every* validator has finalized at least `min_count` @@ -495,9 +497,10 @@ async fn full_node_syncs_from_validators() { }; let peers = build_multiaddrs_excluding(&setups, i); let cfg = build_config_with_role(setup, peers, validator_set.clone(), role); - let svc = MalachiteService::::new(cfg, Arc::clone(&exts[i])) + let mut svc = MalachiteService::::new(cfg, Arc::clone(&exts[i])) .await .expect("service starts"); + svc.start_app_task(); services.push(svc); sleep(Duration::from_millis(500)).await; } diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index 91ee7061e9f..3d9bfa9e112 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -223,6 +223,7 @@ async fn single_validator_finalizes_and_recovers_after_restart() { ) .await .expect("start malachite service"); + svc.start_app_task(); // Feed chain heads one-per-round so the quarantine-advance // probe always sees a strictly newer EB (parent's @@ -265,6 +266,7 @@ async fn single_validator_finalizes_and_recovers_after_restart() { ) .await .expect("restart malachite service"); + svc2.start_app_task(); let mut pending2 = chain[32..].iter().copied(); let (high2, finalized2) = collect_until_finalized(&mut svc2, &mut pending2, 3, Duration::from_secs(60)).await; From 1e49ebffcc9602c4864e9b5cad238a6f7908be40 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 19:05:32 +0300 Subject: [PATCH 47/72] Simplify fast-sync replay filter state --- ethexe/malachite/service/src/externalities.rs | 109 ++++++++++-------- ethexe/malachite/service/src/service.rs | 2 +- 2 files changed, 60 insertions(+), 51 deletions(-) diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index 204b13b3250..eb83fe54b92 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -68,9 +68,9 @@ pub(crate) struct FastSyncReplayTarget { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct FastSyncReplayFilter { - pub target: FastSyncReplayTarget, - pub target_proposal_seen: bool, +pub(crate) enum FastSyncReplayFilter { + UntilTargetProposal(FastSyncReplayTarget), + UntilTargetFinalization { mb_hash: H256 }, } /// Inputs the externalities need to satisfy the [`ethexe_malachite_core::Externalities`] @@ -114,9 +114,9 @@ pub(crate) struct EthexeExternalities { pub(crate) post_quarantine_delay: u32, /// Fast-sync startup gate, not a general live/replay classifier. /// While active it suppresses all Malachite callbacks until the - /// captured boundary proposal and finalization are replayed. Callers + /// captured target proposal and finalization are replayed. Callers /// must enable it before starting the Malachite application task. - pub(crate) fast_sync_replay_filter: RwLock>, + pub(crate) fast_sync_replay_filter: Mutex>, } /// One outbound [`MalachiteEvent`] that can't be released until its @@ -640,11 +640,10 @@ impl EthexeExternalities { pub(crate) fn enable_fast_sync_replay_filter(&self, mb_hash: H256, eb_hash: H256) { *self .fast_sync_replay_filter - .write() - .expect("fast_sync_replay_filter poisoned") = Some(FastSyncReplayFilter { - target: FastSyncReplayTarget { mb_hash, eb_hash }, - target_proposal_seen: false, - }); + .lock() + .expect("fast_sync_replay_filter poisoned") = Some( + FastSyncReplayFilter::UntilTargetProposal(FastSyncReplayTarget { mb_hash, eb_hash }), + ); } fn suppress_for_fast_sync_replay_proposal( @@ -654,53 +653,55 @@ impl EthexeExternalities { ) -> Result { let mut guard = self .fast_sync_replay_filter - .write() + .lock() .expect("fast_sync_replay_filter poisoned"); - let Some(filter) = *guard else { - return Ok(false); - }; + match *guard { + None => Ok(false), + Some(FastSyncReplayFilter::UntilTargetProposal(target)) => { + let matches_target = + mb_hash == target.mb_hash && last_advanced_eb == target.eb_hash; - let target = filter.target; - let matches_target = mb_hash == target.mb_hash && last_advanced_eb == target.eb_hash; + if mb_hash == target.mb_hash && !matches_target { + let message = format!( + "fast-sync replay target mismatch: MB {mb_hash} advanced {last_advanced_eb}, expected {}", + target.eb_hash, + ); + *guard = None; + drop(guard); - if mb_hash == target.mb_hash && !matches_target { - let message = format!( - "fast-sync replay target mismatch: MB {mb_hash} advanced {last_advanced_eb}, expected {}", - target.eb_hash, - ); - *guard = None; - drop(guard); + let _ = self.event_tx.send(Err(anyhow!(message.clone()))); + return Err(anyhow!(message)); + } - let _ = self.event_tx.send(Err(anyhow!(message.clone()))); - return Err(anyhow!(message)); - } + if matches_target { + *guard = Some(FastSyncReplayFilter::UntilTargetFinalization { mb_hash }); + } - if matches_target { - guard - .as_mut() - .expect("fast-sync replay filter checked above") - .target_proposal_seen = true; + Ok(true) + } + Some(FastSyncReplayFilter::UntilTargetFinalization { .. }) => Ok(true), } - - Ok(true) } fn suppress_for_fast_sync_replay_finalization(&self, mb_hash: H256) -> bool { let mut guard = self .fast_sync_replay_filter - .write() + .lock() .expect("fast_sync_replay_filter poisoned"); - let Some(filter) = *guard else { - return false; - }; - - if mb_hash == filter.target.mb_hash && filter.target_proposal_seen { - *guard = None; + match *guard { + None => false, + Some(FastSyncReplayFilter::UntilTargetProposal(_)) => true, + Some(FastSyncReplayFilter::UntilTargetFinalization { + mb_hash: target_mb_hash, + }) => { + if mb_hash == target_mb_hash { + *guard = None; + } + true + } } - - true } /// True iff `prerequisite.is_zero()` (no prerequisite — genesis @@ -879,7 +880,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 0, post_quarantine_delay: 0, - fast_sync_replay_filter: RwLock::new(None), + fast_sync_replay_filter: Mutex::new(None), }; (ext, event_rx) } @@ -1290,7 +1291,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 0, post_quarantine_delay: 0, - fast_sync_replay_filter: RwLock::new(None), + fast_sync_replay_filter: Mutex::new(None), }; let payload = Operations::new(vec![ @@ -1347,6 +1348,14 @@ mod tests { .await .unwrap(); assert!(rx.try_recv().is_err(), "boundary proposal is suppressed"); + assert_eq!( + *ext.fast_sync_replay_filter + .lock() + .expect("fast_sync_replay_filter poisoned"), + Some(FastSyncReplayFilter::UntilTargetFinalization { + mb_hash: boundary_hash, + }), + ); assert!( ext.pending_events .lock() @@ -1435,7 +1444,7 @@ mod tests { ); assert!( ext.fast_sync_replay_filter - .read() + .lock() .expect("fast_sync_replay_filter poisoned") .is_some(), "filter must stay active until boundary proposal is seen", @@ -1461,7 +1470,7 @@ mod tests { .unwrap(); assert!( ext.fast_sync_replay_filter - .read() + .lock() .expect("fast_sync_replay_filter poisoned") .is_none(), "boundary proposal plus finalization clears filter", @@ -1511,7 +1520,7 @@ mod tests { } assert!( ext.fast_sync_replay_filter - .read() + .lock() .expect("fast_sync_replay_filter poisoned") .is_none(), "mismatch must clear replay filter", @@ -1546,7 +1555,7 @@ mod tests { } assert!( ext.fast_sync_replay_filter - .read() + .lock() .expect("fast_sync_replay_filter poisoned") .is_none(), "mismatch must clear replay filter", @@ -1580,7 +1589,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 0, post_quarantine_delay: 0, - fast_sync_replay_filter: RwLock::new(None), + fast_sync_replay_filter: Mutex::new(None), }; (ext, event_rx) } @@ -2332,7 +2341,7 @@ mod tests { gas_allowance: 1_000_000, canonical_quarantine: 2, post_quarantine_delay: 3, - fast_sync_replay_filter: RwLock::new(None), + fast_sync_replay_filter: Mutex::new(None), }; let candidate = ext diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 0a85016f8d7..f14816e0858 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -165,7 +165,7 @@ impl MalachiteService { gas_allowance: config.gas_allowance, canonical_quarantine: config.canonical_quarantine, post_quarantine_delay: config.post_quarantine_delay, - fast_sync_replay_filter: std::sync::RwLock::new(None), + fast_sync_replay_filter: std::sync::Mutex::new(None), }); // On-chain addresses → pub keys, so era rotations resolve back without an out-of-band lookup. From 788c466a36f90cb86b3b83a2145de6785a73b320 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 19:38:41 +0300 Subject: [PATCH 48/72] Address fast-sync replay review --- ethexe/malachite/core/src/service.rs | 3 ++ ethexe/malachite/service/src/externalities.rs | 38 +++++++++++++----- ethexe/malachite/service/src/lib.rs | 1 + ethexe/malachite/service/src/service.rs | 14 +++---- ethexe/network/src/lib.rs | 40 +++---------------- ethexe/service/src/fast_sync.rs | 16 ++------ ethexe/service/src/tests/utils/env.rs | 12 ++---- 7 files changed, 54 insertions(+), 70 deletions(-) diff --git a/ethexe/malachite/core/src/service.rs b/ethexe/malachite/core/src/service.rs index 9818851440d..261e89bbe40 100644 --- a/ethexe/malachite/core/src/service.rs +++ b/ethexe/malachite/core/src/service.rs @@ -372,6 +372,9 @@ fn build_inner_config(cfg: &MalachiteConfig, moniker: &str) -> InnerNodeConfig { let value_sync = match cfg.env { Environment::Production => ValueSyncConfig::default(), Environment::Test => ValueSyncConfig { + // Service tests do not run Malachite's application task forever, so + // use short value-sync waits and small batches to make missing replay + // data fail fast instead of blocking the event queue. status_update_interval: Duration::from_millis(500), request_timeout: Duration::from_secs(3), parallel_requests: 16, diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index eb83fe54b92..1e53c059074 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -62,7 +62,7 @@ use tokio::sync::{Notify, mpsc}; use tracing::{error, info, warn}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct FastSyncReplayTarget { +pub struct FastSyncReplayTarget { pub mb_hash: H256, pub eb_hash: H256, } @@ -179,6 +179,9 @@ impl Externalities for EthexeExternalities { meta.last_advanced_eb = last_advanced; }); + // The fast-sync replay gate sits after durable MB writes so a suppressed + // replayed proposal is still available to finalization and DB walks; only + // outbound events are skipped. if self.suppress_for_fast_sync_replay_proposal(mb_hash, last_advanced)? { return Ok(()); } @@ -211,6 +214,9 @@ impl Externalities for EthexeExternalities { ) })?; + // The replay gate sits after loading the compact block and operations: + // even suppressed finalizations must prove the proposal path persisted + // the MB before we skip mempool, globals, and event side effects. if self.suppress_for_fast_sync_replay_finalization(mb_hash) { return Ok(()); } @@ -637,13 +643,12 @@ impl Externalities for EthexeExternalities { } impl EthexeExternalities { - pub(crate) fn enable_fast_sync_replay_filter(&self, mb_hash: H256, eb_hash: H256) { + pub(crate) fn enable_fast_sync_replay_filter(&self, target: FastSyncReplayTarget) { *self .fast_sync_replay_filter .lock() - .expect("fast_sync_replay_filter poisoned") = Some( - FastSyncReplayFilter::UntilTargetProposal(FastSyncReplayTarget { mb_hash, eb_hash }), - ); + .expect("fast_sync_replay_filter poisoned") = + Some(FastSyncReplayFilter::UntilTargetProposal(target)); } fn suppress_for_fast_sync_replay_proposal( @@ -663,6 +668,9 @@ impl EthexeExternalities { mb_hash == target.mb_hash && last_advanced_eb == target.eb_hash; if mb_hash == target.mb_hash && !matches_target { + // Fast sync captures the on-chain committed MB/EB as one + // event pair. Replaying the target MB with another advanced + // EB means the restored anchor is internally inconsistent. let message = format!( "fast-sync replay target mismatch: MB {mb_hash} advanced {last_advanced_eb}, expected {}", target.eb_hash, @@ -1342,7 +1350,10 @@ mod tests { let boundary_payload = payload(Some(boundary_eb), 1); let boundary_block = wrap(boundary_payload, 10, H256::zero()); let boundary_hash = boundary_block.hash(); - ext.enable_fast_sync_replay_filter(boundary_hash, boundary_eb); + ext.enable_fast_sync_replay_filter(FastSyncReplayTarget { + mb_hash: boundary_hash, + eb_hash: boundary_eb, + }); ext.process_mb_proposal(boundary_hash, boundary_block) .await @@ -1423,7 +1434,10 @@ mod tests { let boundary_payload = payload(Some(boundary_eb), 1); let boundary_block = wrap(boundary_payload.clone(), 10, H256::zero()); let boundary_hash = boundary_block.hash(); - ext.enable_fast_sync_replay_filter(boundary_hash, boundary_eb); + ext.enable_fast_sync_replay_filter(FastSyncReplayTarget { + mb_hash: boundary_hash, + eb_hash: boundary_eb, + }); let operations_hash = db.set_operations(boundary_payload); db.set_mb_compact_block( @@ -1501,7 +1515,10 @@ mod tests { let actual_eb = H256::repeat_byte(0xB2); let block = wrap(payload(Some(actual_eb), 1), 10, H256::zero()); let mb_hash = block.hash(); - ext.enable_fast_sync_replay_filter(mb_hash, expected_eb); + ext.enable_fast_sync_replay_filter(FastSyncReplayTarget { + mb_hash, + eb_hash: expected_eb, + }); let err = ext .process_mb_proposal(mb_hash, block) @@ -1536,7 +1553,10 @@ mod tests { let actual_eb = H256::repeat_byte(0xC1); let block = wrap(payload(Some(actual_eb), 1), 10, H256::zero()); let mb_hash = block.hash(); - ext.enable_fast_sync_replay_filter(mb_hash, expected_eb); + ext.enable_fast_sync_replay_filter(FastSyncReplayTarget { + mb_hash, + eb_hash: expected_eb, + }); let err = ext .process_mb_proposal(mb_hash, block) diff --git a/ethexe/malachite/service/src/lib.rs b/ethexe/malachite/service/src/lib.rs index e7f2d8aa324..b5bb7e3befe 100644 --- a/ethexe/malachite/service/src/lib.rs +++ b/ethexe/malachite/service/src/lib.rs @@ -57,6 +57,7 @@ mod tx_validity; pub use crate::{ config::{MalachiteConfig, ValidatorEntry}, + externalities::FastSyncReplayTarget, mempool::{DEFAULT_POOL_CAPACITY, InjectedTxMempool, Mempool, TxInsertionStatus}, service::MalachiteService, tx_validity::{MIN_EXECUTABLE_BALANCE_FOR_INJECTED_MESSAGES, TxValidity, TxValidityChecker}, diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index f14816e0858..4d7610575a3 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -36,7 +36,8 @@ use gsigner::{Signer, schemes::secp256k1::Secp256k1}; use tokio::sync::{Notify, mpsc}; use crate::{ - MalachiteConfig, MalachiteEvent, Mempool, ValidatorEntry, externalities::EthexeExternalities, + FastSyncReplayTarget, MalachiteConfig, MalachiteEvent, Mempool, ValidatorEntry, + externalities::EthexeExternalities, }; /// Public consensus service. @@ -198,23 +199,22 @@ impl MalachiteService { } } - pub fn enable_fast_sync_replay_filter(&self, mb_hash: H256, eb_hash: H256) -> Result { + pub fn enable_fast_sync_replay_filter(&self, target: FastSyncReplayTarget) -> Result { if self .inner .as_ref() .context("Malachite inner service is shut down")? - .is_finalized(mb_hash)? + .is_finalized(target.mb_hash)? { tracing::info!( - mb_hash = %mb_hash, - eb_hash = %eb_hash, + mb_hash = %target.mb_hash, + eb_hash = %target.eb_hash, "not enabling fast-sync replay filter for already-finalized MB", ); return Ok(false); } - self.externalities - .enable_fast_sync_replay_filter(mb_hash, eb_hash); + self.externalities.enable_fast_sync_replay_filter(target); Ok(true) } diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index 0cfaec3a726..7a2145d0882 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -59,10 +59,7 @@ use libp2p::{ metrics::Recorder, multiaddr::Protocol, ping, - swarm::{ - Config as SwarmConfig, NetworkBehaviour, SwarmEvent, behaviour::toggle::Toggle, - dial_opts::DialOpts, - }, + swarm::{Config as SwarmConfig, NetworkBehaviour, SwarmEvent, behaviour::toggle::Toggle}, yamux, }; #[cfg(test)] @@ -315,25 +312,8 @@ impl NetworkService { }) .context("bootstrap nodes are not allowed without peer ID")?; - let mut peer_addr = Multiaddr::empty(); - for protocol in multiaddr.iter() { - if !matches!(protocol, Protocol::P2p(_)) { - peer_addr.push(protocol); - } - } - - swarm.add_peer_address(peer_id, peer_addr.clone()); - swarm - .behaviour_mut() - .kad - .add_address(peer_id, peer_addr.clone()); - if let Err(error) = swarm.dial( - DialOpts::peer_id(peer_id) - .addresses(vec![peer_addr]) - .build(), - ) { - log::warn!("failed to dial bootstrap peer {peer_id}: {error}"); - } + swarm.add_peer_address(peer_id, multiaddr.clone()); + swarm.behaviour_mut().kad.add_address(peer_id, multiaddr); bootstrap_peers.insert(peer_id); } @@ -863,11 +843,6 @@ mod tests { } } - fn bootstrap_address(mut self, address: Multiaddr) -> Self { - self.bootstrap_addresses.insert(address); - self - } - fn build(self) -> NetworkService { const GENESIS_BLOCK_HEADER: BlockHeader = BlockHeader { height: 0, @@ -956,18 +931,15 @@ mod tests { } #[tokio::test] - async fn request_db_data_from_bootstrap_peer() { + async fn request_db_data_from_connected_peer() { init_logger(); let service2 = NetworkServiceBuilder::new(); let hello = service2.db.cas().write(b"hello"); let mut service2 = service2.build(); - let service2_peer_id = service2.local_peer_id(); - let (service2_addr, _) = service2.swarm.listen().with_memory_addr_external().await; - let service2_addr = service2_addr.with_p2p(service2_peer_id).unwrap(); - let service1 = NetworkServiceBuilder::new().bootstrap_address(service2_addr); - let service1 = service1.build(); + let mut service1 = NetworkServiceBuilder::new().build(); + service1.connect(&mut service2).await; let service1_handle = service1.bitswap_handle(); tokio::spawn(service1.loop_on_next()); diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 43111d4a68d..7c8a24410a5 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -29,6 +29,7 @@ use ethexe_db::{ visitor::DatabaseVisitor, }; use ethexe_ethereum::{mirror::MirrorQuery, router::RouterQuery}; +use ethexe_malachite::FastSyncReplayTarget; use ethexe_network::NetworkService; use ethexe_observer::{ObserverService, utils::BlockLoader}; use ethexe_runtime_common::{ @@ -46,15 +47,9 @@ use std::{ collections::{BTreeMap, BTreeSet, HashMap}, }; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct CommittedChain { - pub mb_hash: H256, - pub eb_hash: H256, -} - struct EventData { latest_committed_batch: Digest, - latest_committed_chain: CommittedChain, + latest_committed_chain: FastSyncReplayTarget, } impl EventData { @@ -82,7 +77,7 @@ impl EventData { pending_eb = Some(*eb_hash); } BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb_hash))) => { - latest_committed_chain.get_or_insert(CommittedChain { + latest_committed_chain.get_or_insert(FastSyncReplayTarget { mb_hash: *mb_hash, eb_hash: pending_eb.take().unwrap_or_default(), }); @@ -747,10 +742,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { if let Some(malachite) = malachite.as_mut() { // `Service::run` performs fast sync before `run_inner().start_app_task()`, // so live Malachite callbacks cannot race this startup replay gate. - let _ = malachite.enable_fast_sync_replay_filter( - latest_committed_chain.mb_hash, - latest_committed_chain.eb_hash, - )?; + let _ = malachite.enable_fast_sync_replay_filter(latest_committed_chain)?; malachite.receive_new_chain_head(block_data.to_simple()); } diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index 973d9247b41..3045ed20fea 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -39,9 +39,9 @@ use ethexe_ethereum::{ router::RouterQuery, }; use ethexe_malachite::{ - InjectedTxMempool, MalachiteConfig, MalachiteConfigEnvironment, MalachiteService, - Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, derive_libp2p_secret, - malachite_libp2p_peer_id, + FastSyncReplayTarget, InjectedTxMempool, MalachiteConfig, MalachiteConfigEnvironment, + MalachiteService, Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, + derive_libp2p_secret, malachite_libp2p_peer_id, }; use ethexe_network::{NetworkConfig, NetworkRuntimeConfig, NetworkService, export::Multiaddr}; use ethexe_observer::{ @@ -1004,11 +1004,7 @@ impl Wallets { } } -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub struct LatestFastSyncedBlocks { - pub eb_hash: H256, - pub mb_hash: H256, -} +pub type LatestFastSyncedBlocks = FastSyncReplayTarget; pub struct Node { pub name: Option, From d9f63d2087c9b2d12d017ae23af924fdabc6c949 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 19:43:52 +0300 Subject: [PATCH 49/72] Clean up network test builder --- ethexe/network/src/lib.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index 7a2145d0882..925698967ae 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -829,7 +829,6 @@ mod tests { latest_validators: ValidatorsVec, signer: Signer, validator_key: Option, - bootstrap_addresses: HashSet, } impl NetworkServiceBuilder { @@ -839,7 +838,6 @@ mod tests { latest_validators: nonempty![Address::default()].into(), signer: Signer::memory(), validator_key: None, - bootstrap_addresses: HashSet::new(), } } @@ -861,7 +859,6 @@ mod tests { latest_validators, signer, validator_key, - bootstrap_addresses, } = self; db.set_config(DBConfig { @@ -870,8 +867,7 @@ mod tests { }); let key = signer.generate().unwrap(); - let mut config = NetworkConfig::new_test(key, Address::default()); - config.bootstrap_addresses = bootstrap_addresses; + let config = NetworkConfig::new_test(key, Address::default()); let runtime_config = NetworkRuntimeConfig { latest_block_header: GENESIS_BLOCK_HEADER, From a4f06caf5db5b9b8bc0a04e3fa520bbe8a9f2af8 Mon Sep 17 00:00:00 2001 From: Grisha Sobol Date: Thu, 11 Jun 2026 18:44:53 +0200 Subject: [PATCH 50/72] fix(vara.eth/malachite): address codex review, stabilize heavy tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - app loop: propagate fatal errors again — the catch-all made `FinalizationError::Fatal` dead code and silenced the error stream - receive_eb_synced: wake the producer on stale synced blocks too (a lower-height sync may land headers a failed descendant walk needs) - restore the `candidate == parent_advance` short-circuit in find_eb_candidate_for_advancing (no more "X does not descend from X" warn spam every idle round) - drop the redundant compute_mb on BlockFinalized: BlockProposal is always emitted first on every node, so compute is already triggered - make propose_timeout a MalachiteServiceConfig field (default stays 2 * SLOT_DURATION — Ethereum block time can stretch past one slot); test envs use a short timeout so idle rounds don't burn 24s each - MalachiteServiceStarter::new: drop unused async - nextest: reserve 4 threads for ethexe-service tests in the default profile like CI does — heavy multi-validator tests starved under full-suite parallelism and blew the 120s cap (full local suite is 527/527 with this) Co-Authored-By: Claude Opus 4.7 --- .config/nextest.toml | 3 +++ ethexe/malachite/core/src/app.rs | 9 ++++++--- ethexe/malachite/service/src/config.rs | 11 ++++++++++- ethexe/malachite/service/src/externalities.rs | 5 +++++ ethexe/malachite/service/src/service.rs | 6 +++++- ethexe/malachite/service/src/starter.rs | 4 ++-- .../service/tests/restart_resilience.rs | 3 +-- ethexe/service/src/lib.rs | 17 ++++------------- ethexe/service/src/tests/utils/env.rs | 4 +++- 9 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 5b47a1d07c0..329edbd6f48 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -27,6 +27,9 @@ test-group = 'authorship' [[profile.default.overrides]] filter = 'package(ethexe-service)' leak-timeout = { period = "10s", result = "pass" } +# Heavy multi-validator tests (4 Anvil + 4 malachite engines) starve under +# full-suite parallelism and blow the 120s cap; reserve threads like CI does. +threads-required = 4 [profile.default.junit] path = "junit.xml" diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index 691c0d60098..fa1a9bf9376 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -117,9 +117,12 @@ where return Err(anyhow!("consensus channel closed")); }; - if let Err(err) = self.handle_app_msg(msg).await { - error!("error handling AppMsg: {err}"); - } + // `Err` from `handle_app_msg` means a FATAL failure (dropped + // engine reply channel, `FinalizationError::Fatal`, broken + // proposal production) — propagate so the app task terminates + // and the error surfaces on the service stream. Non-fatal + // cases are logged and absorbed inside `handle_app_msg`. + self.handle_app_msg(msg).await?; } } diff --git a/ethexe/malachite/service/src/config.rs b/ethexe/malachite/service/src/config.rs index 2ab6ec1809c..e177b259294 100644 --- a/ethexe/malachite/service/src/config.rs +++ b/ethexe/malachite/service/src/config.rs @@ -11,7 +11,7 @@ use crate::Mempool; use ethexe_common::ecdsa::{PublicKey, Signer}; pub use ethexe_malachite_core::{Multiaddr, ValidatorEntry}; -use std::{net::SocketAddr, path::PathBuf}; +use std::{net::SocketAddr, path::PathBuf, time::Duration}; #[derive(Clone, Debug)] pub struct MalachiteServiceConfig { @@ -43,6 +43,10 @@ pub struct MalachiteServiceConfig { /// The complete validator set. Quorum is `> 2/3` of total voting power. /// A validator node's own public key must appear in this list. pub validators: Vec, + + /// How long the proposer may wait for proposable content before the + /// round times out and rotates. + pub propose_timeout: Duration, } impl MalachiteServiceConfig { @@ -56,6 +60,10 @@ impl MalachiteServiceConfig { std::net::IpAddr::V4(std::net::Ipv4Addr::new(0, 0, 0, 0)), 20334, ); + /// Ethereum block time occasionally stretches past one slot, so give + /// the producer more than `SLOT_DURATION` to find a fresh EB. + pub const DEFAULT_PROPOSE_TIMEOUT: Duration = + Duration::from_secs(2 * alloy::eips::merge::SLOT_DURATION.as_secs()); /// Build a config with defaults from the node's home directory. /// The validator set is left empty — fill it in with [`Self::with_validators`]. @@ -68,6 +76,7 @@ impl MalachiteServiceConfig { home_dir, persistent_peers: Vec::new(), validators: Vec::new(), + propose_timeout: Self::DEFAULT_PROPOSE_TIMEOUT, } } diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index cc8503a9b62..51c23377182 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -537,6 +537,11 @@ impl EthexeExternalities { Err(e) => return Err(anyhow!("quarantine anchor lookup failed: {e}")), }; + if candidate.hash == parent_advance { + // No new EB past quarantine since the parent's advance. + return Ok(None); + } + match quarantine::is_strict_descendant_of(&self.db, candidate, parent_advance, start) { Ok(Acceptance::Accepted(())) => Ok(Some(candidate.hash)), Ok(Acceptance::Rejected(reason)) => { diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 8254a4e4ab3..87ddb73508b 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -95,8 +95,12 @@ impl MalachiteService { tracing::trace!( latest_synced = %*latest_synced, synced = %synced, - "synced EB is not newer than the current synced head, ignoring" + "synced EB is not newer than the current synced head" ); + drop(latest_synced); + // Still wake the producer: a lower-height sync may have just + // landed parent headers a failed descendant walk needs. + self.chain_head.notify.notify_one(); return; } *latest_synced = synced; diff --git a/ethexe/malachite/service/src/starter.rs b/ethexe/malachite/service/src/starter.rs index d887314b449..e2da525813e 100644 --- a/ethexe/malachite/service/src/starter.rs +++ b/ethexe/malachite/service/src/starter.rs @@ -37,7 +37,7 @@ impl MalachiteServiceStarter { /// Prepare a service: resolve the node role (validator / full node), /// build the externalities and the core config. /// Fails if `config.validators` is empty or the validator key is absent in the signer. - pub async fn new( + pub fn new( config: MalachiteServiceConfig, validator_config: Option>, db: Database, @@ -85,7 +85,7 @@ impl MalachiteServiceStarter { validator_secret, validators: config.validators.clone(), role, - propose_timeout: alloy::eips::merge::SLOT_DURATION * 2, + propose_timeout: config.propose_timeout, }; let chain_head = Arc::new(ChainHead { diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index f370c0d78cc..98c72f0d7b2 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -138,6 +138,7 @@ fn build_config( public_key: pub_key, voting_power: 1, }], + propose_timeout: Duration::from_secs(5), } } @@ -232,7 +233,6 @@ async fn single_validator_finalizes_and_recovers_after_restart() { db.clone(), chain[0], ) - .await .expect("create malachite service starter") .start() .await @@ -280,7 +280,6 @@ async fn single_validator_finalizes_and_recovers_after_restart() { db.clone(), chain[31], ) - .await .expect("create malachite service starter after restart") .start() .await diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index cede124539e..0fd65620c80 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -540,7 +540,6 @@ impl Service { db.clone(), initial_chain_head, ) - .await .context("failed to create Malachite service starter")? }; @@ -982,18 +981,10 @@ impl Service { "Malachite: BlockFinalized", ); - // +_+_+ ??? whether we really need this? - // Non-proposer nodes (validators that didn't propose - // this height + every full/RPC node) first see the MB - // here. Trigger compute so the body — including any - // injected-tx `Promise` — is produced locally; the - // matching `SignedCompactPromise` arrives via the - // network and is joined into a full `SignedTxReceipt` - // by the RPC subscription manager. Calls are - // idempotent: a proposer that already computed via - // `BlockProposal` short-circuits on - // `mb_meta.computed`. - compute.compute_mb(mb_hash, ethexe_common::PromisePolicy::Enabled); + // No compute here: `BlockProposal` is always emitted + // before the matching `BlockFinalized` (on every node, + // including the sync path), so compute for this MB has + // already been triggered. } MalachiteEvent::PurgedTransactions { eb_hash, diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index fddeb3ef641..455f1479656 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -1195,6 +1195,9 @@ impl Node { .with_validators(validators); mc.canonical_quarantine = self.canonical_quarantine; mc.post_quarantine_delay = self.post_quarantine_delay; + // Tests drive content in bursts; a short propose timeout keeps + // idle rounds cheap instead of burning a 24s slot each. + mc.propose_timeout = Duration::from_secs(3); let malachite_validator_config = self.validator_config @@ -1214,7 +1217,6 @@ impl Node { self.db.clone(), latest_block, ) - .await .expect("MalachiteServiceStarter::new") }; From 6c2ecb6f7c6c266ffe01cb41663eaf62b1d864f4 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Thu, 11 Jun 2026 19:54:42 +0300 Subject: [PATCH 51/72] Simplify fast-sync test event data --- ethexe/service/src/fast_sync.rs | 10 +++-- ethexe/service/src/tests/utils/env.rs | 49 +++++++++--------------- ethexe/service/src/tests/utils/events.rs | 8 +++- 3 files changed, 31 insertions(+), 36 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 7c8a24410a5..7e330da096d 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -753,10 +753,12 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { #[cfg(test)] sender - .send(crate::tests::utils::TestingEvent::FastSyncDone { - eb_hash: latest_committed_eb, - mb_hash: latest_committed_mb, - }) + .send(crate::tests::utils::TestingEvent::FastSyncDone( + crate::tests::utils::LatestFastSyncedBlocks { + eb_hash: latest_committed_eb, + mb_hash: latest_committed_mb, + }, + )) .await; Ok(()) diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index 3045ed20fea..f9ac1d81f9d 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -6,7 +6,10 @@ use crate::{ config::EthereumConfig, tests::utils::{ InfiniteStreamExt, TestingEvent, TestingNetworkEvent, - events::{self, ObserverEventReceiver, ObserverEventSender, TestingEventReceiver}, + events::{ + self, LatestFastSyncedBlocks, ObserverEventReceiver, ObserverEventSender, + TestingEventReceiver, + }, }, }; use alloy::{ @@ -39,9 +42,9 @@ use ethexe_ethereum::{ router::RouterQuery, }; use ethexe_malachite::{ - FastSyncReplayTarget, InjectedTxMempool, MalachiteConfig, MalachiteConfigEnvironment, - MalachiteService, Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, - derive_libp2p_secret, malachite_libp2p_peer_id, + InjectedTxMempool, MalachiteConfig, MalachiteConfigEnvironment, MalachiteService, + Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, derive_libp2p_secret, + malachite_libp2p_peer_id, }; use ethexe_network::{NetworkConfig, NetworkRuntimeConfig, NetworkService, export::Multiaddr}; use ethexe_observer::{ @@ -1004,8 +1007,6 @@ impl Wallets { } } -pub type LatestFastSyncedBlocks = FastSyncReplayTarget; - pub struct Node { pub name: Option, pub db: Database, @@ -1271,34 +1272,20 @@ impl Node { self.running_service_handle = Some(handle); self.shutdown_tx = Some(shutdown_tx); - let mut service_started = false; if self.fast_sync { - match self - .events() - .find(|event| { - matches!( - event, - TestingEvent::FastSyncDone { .. } | TestingEvent::ServiceStarted - ) - }) - .await - { - TestingEvent::FastSyncDone { eb_hash, mb_hash } => { - self.latest_fast_synced_blocks = - Some(LatestFastSyncedBlocks { eb_hash, mb_hash }); - } - TestingEvent::ServiceStarted => { - service_started = true; - } - _ => unreachable!("filter only allows fast-sync or service-started events"), - } + self.latest_fast_synced_blocks = Some( + self.events() + .find_map(|event| match event { + TestingEvent::FastSyncDone(blocks) => Some(blocks), + _ => None, + }) + .await, + ); } - if !service_started { - self.events() - .find(|e| matches!(e, TestingEvent::ServiceStarted)) - .await; - } + self.events() + .find(|e| matches!(e, TestingEvent::ServiceStarted)) + .await; // fast sync implies network has connections if wait_for_network && !self.fast_sync { diff --git a/ethexe/service/src/tests/utils/events.rs b/ethexe/service/src/tests/utils/events.rs index e997e9da052..55011bbe0ad 100644 --- a/ethexe/service/src/tests/utils/events.rs +++ b/ethexe/service/src/tests/utils/events.rs @@ -42,6 +42,12 @@ pub type TestingEventReceiver = KickingStream>; pub type ObserverEventSender = EventSender; pub type ObserverEventReceiver = KickingStream>; +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub struct LatestFastSyncedBlocks { + pub eb_hash: H256, + pub mb_hash: H256, +} + #[derive(Debug, Clone, Eq, PartialEq)] pub enum TestingNetworkInjectedEvent { InboundTransaction { @@ -124,7 +130,7 @@ impl TestingRpcEvent { #[derive(Debug, Clone, Eq, PartialEq)] pub enum TestingEvent { // Fast sync done. Sent just once. - FastSyncDone { eb_hash: H256, mb_hash: H256 }, + FastSyncDone(LatestFastSyncedBlocks), // Basic event to notify that service has started. Sent just once. ServiceStarted, // Services events. From 56d3730c0408309ff1cc360a259618c52a7d2196 Mon Sep 17 00:00:00 2001 From: Gregory Sobol Date: Fri, 12 Jun 2026 15:22:48 +0200 Subject: [PATCH 52/72] fix problem with collect_advance_chain --- ethexe/compute/src/compute.rs | 245 +++++++++++++++++++++---------- ethexe/compute/src/lib.rs | 9 +- ethexe/compute/src/service.rs | 19 ++- ethexe/db/src/migrations/init.rs | 2 +- 4 files changed, 185 insertions(+), 90 deletions(-) diff --git a/ethexe/compute/src/compute.rs b/ethexe/compute/src/compute.rs index 970f7d30da9..91fb33c9967 100644 --- a/ethexe/compute/src/compute.rs +++ b/ethexe/compute/src/compute.rs @@ -11,8 +11,11 @@ use crate::{ComputeError, ComputeEvent, ProcessorExt, Result, service::SubService}; use ethexe_common::{ - PromiseEmissionMode, PromisePolicy, - db::{CodesStorageRW, CompactMb, ConfigStorageRO, MbStorageRO, MbStorageRW, OnChainStorageRO}, + PromiseEmissionMode, PromisePolicy, SimpleBlockData, + db::{ + CodesStorageRW, CompactMb, ConfigStorageRO, GlobalsStorageRO, MbStorageRO, MbStorageRW, + OnChainStorageRO, + }, events::BlockRequestEvent, injected::Promise, malachite::{Operation, Operations}, @@ -241,15 +244,9 @@ pub fn prepare_executable_for_mb( let schedule = db .mb_schedule(parent) .ok_or(ComputeError::ParentMbScheduleMissing(parent))?; - let initial_advanced_block = db.mb_meta(parent).last_advanced_eb; + let advanced_block = db.mb_meta(parent).last_advanced_eb; - build_executable_data( - db, - mb_payload, - program_states, - schedule, - initial_advanced_block, - ) + build_executable_data(db, mb_payload, program_states, schedule, advanced_block) } /// Walk the MB's `Operations` list and prepare processor input. @@ -262,17 +259,28 @@ fn build_executable_data( operations: Operations, program_states: ethexe_common::ProgramStates, schedule: ethexe_common::Schedule, - initial_advanced_block: H256, + advanced_block: H256, ) -> Result { let mut events: Vec = Vec::new(); let mut injected_transactions = Vec::new(); let mut gas_allowance: Option = None; - let mut current_anchor = initial_advanced_block; + + let mut current_anchor = if advanced_block.is_zero() { + None + } else { + Some( + db.block_simple_data(advanced_block) + .ok_or(ComputeError::AnchorBlockHeaderMissing(advanced_block))?, + ) + }; for op in operations.0 { match op { Operation::AdvanceTillEthereumBlock { block_hash } => { - let chain = collect_advance_chain(db, block_hash, current_anchor)?; + let block = db + .block_simple_data(block_hash) + .ok_or(ComputeError::AnchorBlockHeaderMissing(block_hash))?; + let chain = collect_advance_chain(db, block, current_anchor)?; for hash in chain { let block_events = db .block_events(hash) @@ -281,7 +289,7 @@ fn build_executable_data( events.push(event); } } - current_anchor = block_hash; + current_anchor = Some(block); } Operation::Injected(signed) => { let verified = signed.into_verified(); @@ -296,17 +304,19 @@ fn build_executable_data( } } - let anchor_eth_block = if current_anchor.is_zero() { - db.config().genesis_block_hash + let (height, timestamp) = if let Some(current_anchor) = current_anchor { + ( + current_anchor.header.height, + current_anchor.header.timestamp, + ) } else { - current_anchor + db.block_header(db.config().genesis_block_hash) + .map(|h| (h.height, h.timestamp)) + .ok_or(ComputeError::Other( + "genesis block missing from DB; invariant violation", + ))? }; - let (height, timestamp) = db - .block_header(anchor_eth_block) - .map(|h| (h.height, h.timestamp)) - .ok_or(ComputeError::AnchorBlockHeaderMissing(anchor_eth_block))?; - Ok(ExecutableData { height, timestamp, @@ -318,31 +328,69 @@ fn build_executable_data( }) } -/// EBs in `(last_advanced, target]`, oldest-first; capped at 1024. -fn collect_advance_chain(db: &Database, target: H256, last_advanced: H256) -> Result> { - const MAX_ADVANCE_STEPS: usize = 1024; +/// EBs in `(last_advanced, target]`, oldest-first; +fn collect_advance_chain( + db: &Database, + target: SimpleBlockData, + last_advanced: Option, +) -> Result> { + let (last_advanced_hash, last_advanced_height) = if let Some(la) = last_advanced { + (la.hash, la.header.height) + } else { + let start_eb_hash = db.globals().start_block_hash; + let genesis_eb_hash = db.config().genesis_block_hash; + if start_eb_hash != genesis_eb_hash { + return Err(ComputeError::Other( + "if last advanced EB is zero, then start block must match genesis", + )); + } + db.block_simple_data(start_eb_hash) + .ok_or(ComputeError::BlockNotSynced(start_eb_hash)) + .map(|start_eb| { + start_eb + .header + .height + .checked_sub(1) + .ok_or(ComputeError::Other("start block height=0 isn't expected")) + .map(|h| (H256::zero(), h)) + })?? + }; + + let depth = target + .header + .height + .checked_sub(last_advanced_height) + .ok_or(ComputeError::Other("target EB is older than last advanced"))?; - if target == last_advanced { - return Ok(Vec::new()); + if depth == 0 { + return Err(ComputeError::Other( + "target EB is at the same height as last advanced", + )); } - let mut chain = Vec::new(); + let mut chain = Vec::with_capacity(depth as usize); let mut current = target; - while current != last_advanced && current != H256::zero() { - if chain.len() >= MAX_ADVANCE_STEPS { - return Err(ComputeError::AdvanceWalkTooDeep { - target, - last_advanced, - }); + for step in 0..depth { + chain.push(current.hash); + + // The deepest block's parent must connect to the last advanced EB. Its + // parent header is not fetched: for the genesis EB that parent is the + // un-seeded zero hash, so we compare the `parent_hash` field directly. + let parent_hash = current.header.parent_hash; + if step + 1 == depth { + if parent_hash != last_advanced_hash { + return Err(ComputeError::Other( + "collected advancing chain does not connect to the last advanced block", + )); + } + break; } - // Any missing intermediate header has to surface — a silent - // truncation would have validators with different sync depth - // emit different advance chains for the same MB. - let header = db - .block_header(current) - .ok_or(ComputeError::AdvanceMissingHeader { hash: current })?; - chain.push(current); - current = header.parent_hash; + + current = db + .block_simple_data(parent_hash) + .ok_or(ComputeError::Other( + "block header not found while collecting advancing chain", + ))?; } chain.reverse(); @@ -463,21 +511,45 @@ mod tests { use gprimitives::{ActorId, CodeId, MessageId}; use proptest::prelude::*; - fn dummy_ops(db: &Database, tag: u8) -> Operations { - // Tag-derived AdvanceTillEthereumBlock makes each block's - // operations list (and thus its CAS hash) unique across heights. - // The referenced EB also needs a header in the DB so the - // compute-side advance walk picks it up. - let eth_block_hash = H256::from_low_u64_be(0xEB00 + tag as u64); + fn eb_hash(height: u32) -> H256 { + H256::from_low_u64_be(0xEB00 + height as u64) + } + + /// Synthetic Ethereum block at `height`, chained onto its height-1 parent + /// (zero parent at height 1 — the genesis Eth block). The hash is derived + /// from the height, so a contiguous chain builds itself. + fn synthetic_eb(db: &Database, height: u32, events: Vec) -> H256 { + let parent_hash = if height <= 1 { + H256::zero() + } else { + eb_hash(height - 1) + }; + let hash = eb_hash(height); db.set_block_header( - eth_block_hash, + hash, BlockHeader { - height: tag as u32, - timestamp: tag as u64, - parent_hash: H256::zero(), + height, + timestamp: height as u64, + parent_hash, }, ); - db.set_block_events(eth_block_hash, &[]); + db.set_block_events(hash, &events); + hash + } + + /// Seed the genesis Eth block (height 1) and point the genesis zero-MB's + /// `last_advanced_eb` at it, so subsequent MBs walk a real depth-1 chain + /// via the `Some` branch — exactly as the malachite service propagates it. + fn seed_genesis_eth(db: &Database) -> H256 { + let gen_eb = synthetic_eb(db, 1, vec![]); + db.mutate_mb_meta(H256::zero(), |m| m.last_advanced_eb = gen_eb); + gen_eb + } + + fn dummy_ops(db: &Database, eb_height: u32) -> Operations { + // The unique EB height makes each MB's operations list (and thus its + // CAS hash) unique, and gives the advance walk a chained block to pick up. + let eth_block_hash = synthetic_eb(db, eb_height, vec![]); Operations::new(vec![ Operation::AdvanceTillEthereumBlock { block_hash: eth_block_hash, @@ -502,6 +574,13 @@ mod tests { ); } + /// `seed_mb` plus the malachite-side bookkeeping: record the advanced EB + /// as this MB's `last_advanced_eb`, so its child walks a depth-1 chain. + fn seed_mb_advancing(db: &Database, mb_hash: H256, parent: H256, height: u64, eb_height: u32) { + seed_mb(db, mb_hash, parent, height, dummy_ops(db, eb_height)); + db.mutate_mb_meta(mb_hash, |m| m.last_advanced_eb = eb_hash(eb_height)); + } + /// Tail-only queue still computes all uncomputed predecessors. #[tokio::test] #[ntest::timeout(5000)] @@ -510,16 +589,18 @@ mod tests { let db = Database::memory(); seed_genesis_zero_mb(&db); + seed_genesis_eth(&db); let processor = MockProcessor::default(); let mut sub = ComputeSubService::new(db.clone(), processor); - // 5-block chain; mb_hash = 0x1000 + i. + // 5-block chain; mb_hash = 0x1000 + i. Each MB advances one EB, so + // MB at height i pins the EB at height i + 1 (genesis Eth is height 1). const N: u64 = 5; let mut hashes = Vec::with_capacity(N as usize); let mut parent = H256::zero(); for i in 1..=N { let mb_hash = H256::from_low_u64_be(0x1000 + i); - seed_mb(&db, mb_hash, parent, i, dummy_ops(&db, i as u8)); + seed_mb_advancing(&db, mb_hash, parent, i, (i + 1) as u32); hashes.push((i, mb_hash)); parent = mb_hash; } @@ -559,7 +640,7 @@ mod tests { for i in 1..=chain_len { let mb_hash = H256::from_low_u64_be(0xB000 + i); - seed_mb(&db, mb_hash, parent, i, dummy_ops(&db, i as u8)); + seed_mb(&db, mb_hash, parent, i, dummy_ops(&db, (i + 1) as u32)); hashes.push(mb_hash); parent = mb_hash; } @@ -585,15 +666,22 @@ mod tests { #[test] fn collect_advance_chain_errors_on_missing_intermediate_header() { let db = Database::memory(); - let last_advanced = H256::from_low_u64_be(0xA0); + let last_advanced = SimpleBlockData { + hash: H256::from_low_u64_be(0xA0), + header: BlockHeader { + height: 0, + timestamp: 0, + parent_hash: H256::zero(), + }, + }; let parent_b = H256::from_low_u64_be(0xA1); let parent_a = H256::from_low_u64_be(0xA2); - let target = H256::from_low_u64_be(0xA3); + let target_hash = H256::from_low_u64_be(0xA3); // target -> parent_a -> parent_b -> last_advanced // parent_b's header is intentionally missing. db.set_block_header( - target, + target_hash, BlockHeader { height: 3, timestamp: 3, @@ -609,11 +697,16 @@ mod tests { }, ); - let result = collect_advance_chain(&db, target, last_advanced); + let target = db.block_simple_data(target_hash).unwrap(); + let result = collect_advance_chain(&db, target, Some(last_advanced)); match result { - Err(ComputeError::AdvanceMissingHeader { hash }) => assert_eq!(hash, parent_b), + Err(ComputeError::Other(msg)) => assert!( + msg.contains("block header not found"), + "expected a missing-header error for {parent_b:?}, got message: {msg:?} — \ + a silent truncation here would non-determinise event replay across peers" + ), other => panic!( - "expected AdvanceMissingHeader for {parent_b:?}, got {other:?} — \ + "expected a missing-header error for {parent_b:?}, got {other:?} — \ a silent truncation here would non-determinise event replay across peers" ), } @@ -632,7 +725,7 @@ mod tests { let mut sub = ComputeSubService::new(db.clone(), processor); let mb_hash = H256::from_low_u64_be(0xCAFE); - seed_mb(&db, mb_hash, H256::zero(), 1, dummy_ops(&db, 0)); + seed_mb(&db, mb_hash, H256::zero(), 1, dummy_ops(&db, 2)); db.mutate_mb_meta(mb_hash, |meta| { meta.computed = true; }); @@ -679,22 +772,6 @@ mod tests { code_id } - /// Synthetic Ethereum block with a zeroed parent, so the compute-side - /// advance walk collects exactly this single block. - fn synthetic_eb(db: &Database, tag: u8, events: Vec) -> H256 { - let hash = H256::from_low_u64_be(0xEB00 + tag as u64); - db.set_block_header( - hash, - BlockHeader { - height: tag as u32, - timestamp: tag as u64, - parent_hash: H256::zero(), - }, - ); - db.set_block_events(hash, &events); - hash - } - fn ping_injected(destination: ActorId) -> SignedInjectedTransaction { let tx = InjectedTransaction { destination, @@ -730,9 +807,11 @@ mod tests { // MB #0 — create + fund + initialize the ping program via an // Ethereum block. The canonical init message is required: an // injected transaction cannot target an uninitialized program. + // EBs are chained onto the genesis Eth block (height 1), so the + // create block is height 2 and each pinger advances one more EB. let create_eb = synthetic_eb( db, - 0, + 2, vec![ BlockEvent::Router(RouterEvent::ProgramCreated(ProgramCreatedEvent { actor_id: ping_id, @@ -764,11 +843,13 @@ mod tests { }]; ops.extend(mb_bookend()); seed_mb(db, creator, H256::zero(), 0, Operations::new(ops)); + db.mutate_mb_meta(creator, |m| m.last_advanced_eb = create_eb); mb_hashes.push(creator); // MB #1.. — each injects a single PING into the ping program. for i in 1..=pinger_count { - let eb = synthetic_eb(db, i as u8, vec![]); + let eb_height = 2 + i as u32; + let eb = synthetic_eb(db, eb_height, vec![]); let mb_hash = H256::from_low_u64_be(0x1000 + i); let mut ops = vec![ Operation::AdvanceTillEthereumBlock { block_hash: eb }, @@ -782,6 +863,7 @@ mod tests { i, Operations::new(ops), ); + db.mutate_mb_meta(mb_hash, |m| m.last_advanced_eb = eb); mb_hashes.push(mb_hash); } @@ -797,6 +879,7 @@ mod tests { ) -> (Vec, Vec<(H256, Promise)>) { let db = Database::memory(); seed_genesis_zero_mb(&db); + seed_genesis_eth(&db); let mut processor = Processor::new(db.clone()).expect("failed to create processor"); let mb_hashes = build_ping_mb_chain(&db, &mut processor, pinger_count).await; diff --git a/ethexe/compute/src/lib.rs b/ethexe/compute/src/lib.rs index 03a6b3059a7..c404cad21dd 100644 --- a/ethexe/compute/src/lib.rs +++ b/ethexe/compute/src/lib.rs @@ -162,15 +162,12 @@ pub enum ComputeError { AdvanceBlockEventsMissing(H256), #[error("anchor Eth block header missing for {0}")] AnchorBlockHeaderMissing(H256), - #[error("AdvanceTillEthereumBlock walk hit a missing parent header at {hash}")] - AdvanceMissingHeader { hash: H256 }, - #[error( - "AdvanceTillEthereumBlock walk from {target} to {last_advanced} exceeded the safety cap" - )] - AdvanceWalkTooDeep { target: H256, last_advanced: H256 }, #[error(transparent)] Processor(#[from] ProcessorError), + + #[error("other: {0}")] + Other(&'static str), } type Result = std::result::Result; diff --git a/ethexe/compute/src/service.rs b/ethexe/compute/src/service.rs index 30083f8e299..49fc02a618f 100644 --- a/ethexe/compute/src/service.rs +++ b/ethexe/compute/src/service.rs @@ -147,15 +147,30 @@ mod tests { use proptest::{collection, prelude::*}; fn seed_mb(db: &DB, mb_hash: H256, gas_allowance: u64) { - let eth_block_hash = H256::from_low_u64_be(0xEB00); + // Genesis Eth block (height 1) is the root of the advance chain; the + // genesis zero-MB is pinned to it so this MB walks a depth-1 chain. + let genesis_eb = H256::from_low_u64_be(0xEB00); db.set_block_header( - eth_block_hash, + genesis_eb, BlockHeader { height: 1, timestamp: 1, parent_hash: H256::zero(), }, ); + db.set_block_events(genesis_eb, &[]); + db.mutate_mb_meta(H256::zero(), |m| m.last_advanced_eb = genesis_eb); + + // The EB this MB advances to, chained onto the genesis Eth block. + let eth_block_hash = H256::from_low_u64_be(0xEB01); + db.set_block_header( + eth_block_hash, + BlockHeader { + height: 2, + timestamp: 2, + parent_hash: genesis_eb, + }, + ); db.set_block_events(eth_block_hash, &[]); let operations_hash = db.set_operations(Operations::new(vec![ diff --git a/ethexe/db/src/migrations/init.rs b/ethexe/db/src/migrations/init.rs index 36797b30a35..ec12bb575f8 100644 --- a/ethexe/db/src/migrations/init.rs +++ b/ethexe/db/src/migrations/init.rs @@ -85,7 +85,7 @@ pub async fn initialize_empty_db(config: InitConfig, db: &RawDatabase) -> Result let genesis_eb = SimpleBlockData { hash: genesis.hash, header: BlockHeader { - // genesis block header is not important in any way for ethexe + // IMPORTANT: set parent to zero is protocol invariant parent_hash: H256::zero(), height: genesis.number, timestamp: genesis.timestamp, From b3e45469316294452c8d64009c19f5a6c852bf38 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Sun, 14 Jun 2026 14:51:53 +0300 Subject: [PATCH 53/72] Add events module to test utils This change includes the `events` module in the `ethexe/service/src/tests/utils/mod.rs` file. This module is now available for use in tests. --- ethexe/service/src/tests/utils/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ethexe/service/src/tests/utils/mod.rs b/ethexe/service/src/tests/utils/mod.rs index 0b164821cc6..02d07d9de98 100644 --- a/ethexe/service/src/tests/utils/mod.rs +++ b/ethexe/service/src/tests/utils/mod.rs @@ -2,14 +2,14 @@ // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 pub use env::*; -use ethexe_db::{GenesisInitializer, dump::StateDump}; -use ethexe_processor::Processor; pub use events::*; -use futures::FutureExt; mod env; mod events; +use ethexe_db::{GenesisInitializer, dump::StateDump}; +use ethexe_processor::Processor; +use futures::FutureExt; use tracing_subscriber::EnvFilter; pub fn init_logger() { From 17708ef1d80c3938ad01fad240584acc20a78dc7 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Sun, 14 Jun 2026 14:52:19 +0300 Subject: [PATCH 54/72] Cleanup tests --- ethexe/service/src/tests/mod.rs | 394 +------------------------------- 1 file changed, 6 insertions(+), 388 deletions(-) diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 3df3eda4146..8cfb9dbd689 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -3145,91 +3145,9 @@ async fn reply_callback() { #[tokio::test] #[ntest::timeout(120_000)] -async fn fast_sync2() { +async fn fast_sync() { init_logger(); - // let assert_chain = |latest_block, fast_synced_block, alice: &Node, bob: &Node| { - // log::info!("Assert chain in range {latest_block}..{fast_synced_block}"); - // - // IntegrityVerifier::new(alice.db.clone()) - // .verify_chain(latest_block, fast_synced_block) - // .expect("failed to verify Alice database"); - // - // IntegrityVerifier::new(bob.db.clone()) - // .verify_chain(latest_block, fast_synced_block) - // .expect("failed to verify Bob database"); - // - // let alice_globals = alice.db.globals(); - // let bob_globals = bob.db.globals(); - // assert_eq!( - // alice_globals.latest_computed_announce_hash, - // bob_globals.latest_computed_announce_hash - // ); - // assert_eq!( - // alice_globals.latest_prepared_block_hash, - // bob_globals.latest_prepared_block_hash - // ); - // - // let mut block = latest_block; - // loop { - // if fast_synced_block == block { - // break; - // } - // - // log::trace!("assert block {block}"); - // - // // Check block meta, exclude codes_queue and announces, which can vary, and it's ok - // let alice_meta = alice.db.block_meta(block); - // let bob_meta = bob.db.block_meta(block); - // assert!( - // alice_meta.prepared && bob_meta.prepared, - // "Block {block} is not prepared for alice or bob" - // ); - // assert_eq!( - // alice_meta.last_committed_announce, - // bob_meta.last_committed_announce - // ); - // assert_eq!( - // alice_meta.last_committed_batch, - // bob_meta.last_committed_batch - // ); - // - // let alice_announces = alice.db.block_announces(block); - // let bob_announces = bob.db.block_announces(block); - // let Some((alice_announces, bob_announces)) = alice_announces.zip(bob_announces) else { - // panic!("alice or bob has no announces"); - // }; - // - // for &announce_hash in alice_announces.intersection(&bob_announces) { - // if alice.db.announce_meta(announce_hash).computed - // != bob.db.announce_meta(announce_hash).computed - // { - // continue; - // } - // - // assert_eq!( - // alice.db.announce_program_states(announce_hash), - // bob.db.announce_program_states(announce_hash) - // ); - // assert_eq!( - // alice.db.announce_outcome(announce_hash), - // bob.db.announce_outcome(announce_hash) - // ); - // assert_eq!( - // alice.db.announce_outcome(announce_hash), - // bob.db.announce_outcome(announce_hash) - // ); - // } - // - // assert_eq!(alice.db.block_header(block), bob.db.block_header(block)); - // assert_eq!(alice.db.block_events(block), bob.db.block_events(block)); - // assert_eq!(alice.db.block_synced(block), bob.db.block_synced(block)); - // - // let header = alice.db.block_header(block).unwrap(); - // block = header.parent_hash; - // } - // }; - let assert_chain = |latest_block, fast_synced_blocks, alice: &Node, bob: &Node| { let LatestFastSyncedBlocks { eb_hash: fast_synced_eb, @@ -3254,9 +3172,9 @@ async fn fast_sync2() { log::trace!("assert EB {eb_hash}"); - // Check block meta, exclude codes_queue and announces, which can vary, and it's ok let alice_meta = alice.db.block_meta(eb_hash); let bob_meta = bob.db.block_meta(eb_hash); + // we do not compare all fields of meta because they can vary and it's ok assert_eq!(alice_meta.prepared, bob_meta.prepared); assert_eq!( alice_meta.latest_era_validators_committed, @@ -3294,49 +3212,11 @@ async fn fast_sync2() { let alice_schedule = alice.db.mb_schedule(mb_hash); let bob_schedule = bob.db.mb_schedule(mb_hash); assert_eq!(alice_schedule, bob_schedule); - } - // let mut block_hash = latest_block; - // loop { - // if fast_synced_block == block_hash { - // break; - // } - // - // log::trace!("assert MB {mb_hash}"); - // - // // Check block meta, exclude codes_queue and announces, which can vary, and it's ok - // - // let alice_compact_block = alice.db.mb_compact_block(block_hash); - // let bob_compact_block = bob.db.mb_compact_block(block_hash); - // assert_eq!(alice_compact_block, bob_compact_block); - // - // if let (Some(alice_compact_block), Some(bob_compact_block)) = - // (alice_compact_block, bob_compact_block) - // { - // let alice_transactions = - // alice.db.transactions(alice_compact_block.transactions_hash); - // let bob_transactions = alice.db.transactions(bob_compact_block.transactions_hash); - // assert_eq!(alice_transactions, bob_transactions); - // } - // - // let alice_program_states = alice.db.mb_program_states(block_hash); - // let bob_program_states = bob.db.mb_program_states(block_hash); - // assert_eq!(alice_program_states, bob_program_states); - // - // let alice_outcome = alice.db.mb_outcome(block_hash); - // let bob_outcome = alice.db.mb_outcome(block_hash); - // assert_eq!(alice_outcome, bob_outcome); - // - // let alice_schedule = alice.db.mb_schedule(block_hash); - // let bob_schedule = bob.db.mb_schedule(block_hash); - // assert_eq!(alice_schedule, bob_schedule); - // - // let alice_meta = alice.db.mb_meta(block_hash); - // let bob_meta = bob.db.mb_meta(block_hash); - // assert_eq!(alice_meta, bob_meta); - // - // block_hash = alice_compact_block.unwrap().parent; - // } + let alice_meta = alice.db.mb_meta(mb_hash); + let bob_meta = bob.db.mb_meta(mb_hash); + assert_eq!(alice_meta, bob_meta); + } }; let config = TestEnvConfig { @@ -3478,268 +3358,6 @@ async fn fast_sync2() { ); } -#[tokio::test] -#[ntest::timeout(180_000)] -async fn fast_sync() { - init_logger(); - - fn block_prepared(db: ðexe_db::Database, block: H256) -> bool { - db.block_header(block).is_some() - && db.block_events(block).is_some() - && db.block_synced(block) - && db.block_meta(block).prepared - } - - async fn wait_till_block_prepared(receiver: &mut TestingEventReceiver, block: H256) { - if block_prepared(receiver.db(), block) { - return; - } - - receiver - .find_map_with_db(|db, _| block_prepared(&db, block).then_some(())) - .await; - } - - let assert_fast_sync_boundary = |fast_synced_block, alice: &Node, bob: &Node| { - log::info!("Assert fast-sync boundary at {fast_synced_block}"); - - IntegrityVerifier::new(alice.db.clone()) - .verify_chain(fast_synced_block, fast_synced_block) - .expect("failed to verify Alice database"); - - IntegrityVerifier::new(bob.db.clone()) - .verify_chain(fast_synced_block, fast_synced_block) - .expect("failed to verify Bob database"); - - let bob_globals = bob.db.globals(); - let latest_mb = bob_globals.latest_computed_mb_hash; - assert_ne!(latest_mb, H256::zero()); - assert_eq!(bob_globals.latest_finalized_mb_hash, latest_mb); - assert_eq!(bob_globals.latest_prepared_eb_hash, fast_synced_block); - drop(bob_globals); - - let bob_mb_meta = bob.db.mb_meta(latest_mb); - assert!(bob_mb_meta.computed); - assert_eq!(bob_mb_meta.last_advanced_eb, fast_synced_block); - assert!(bob.db.mb_program_states(latest_mb).is_some()); - assert!(bob.db.mb_schedule(latest_mb).is_some()); - assert_eq!(bob.db.mb_outcome(latest_mb), Some(Default::default())); - - if let Some(alice_program_states) = alice.db.mb_program_states(latest_mb) { - assert_eq!( - Some(alice_program_states), - bob.db.mb_program_states(latest_mb) - ); - } - if let Some(alice_schedule) = alice.db.mb_schedule(latest_mb) { - assert_eq!(Some(alice_schedule), bob.db.mb_schedule(latest_mb)); - } - - // Compare the EB/block database surface with Alice while letting - // fast-sync boundary metadata vary, same as the old announce-era test - // allowed codes_queue and announce-specific rows to differ. Bob - // restores these fields at its boundary block instead of replaying - // the later block where Alice observed the commitment. - let alice_meta = alice.db.block_meta(fast_synced_block); - let bob_meta = bob.db.block_meta(fast_synced_block); - assert!( - alice_meta.prepared && bob_meta.prepared, - "Block {fast_synced_block} is not prepared for Alice or Bob" - ); - assert_eq!( - alice_meta.latest_era_validators_committed, - bob_meta.latest_era_validators_committed - ); - - assert_eq!( - alice.db.block_header(fast_synced_block), - bob.db.block_header(fast_synced_block) - ); - assert_eq!( - alice.db.block_events(fast_synced_block), - bob.db.block_events(fast_synced_block) - ); - assert_eq!( - alice.db.block_synced(fast_synced_block), - bob.db.block_synced(fast_synced_block) - ); - - latest_mb - }; - - let assert_chain = - |latest_block, fast_synced_block, fast_synced_mb, alice: &Node, bob: &Node| { - log::info!("Assert chain in range {latest_block}..{fast_synced_block}"); - assert_ne!( - latest_block, fast_synced_block, - "post-fast-sync comparison must cover at least one block" - ); - - IntegrityVerifier::new(alice.db.clone()) - .verify_chain(latest_block, fast_synced_block) - .expect("failed to verify Alice database"); - - let alice_globals = alice.db.globals(); - let bob_globals = bob.db.globals(); - assert_eq!( - alice_globals.latest_prepared_eb_hash, - bob_globals.latest_prepared_eb_hash - ); - - let mut block = latest_block; - loop { - if fast_synced_block == block { - break; - } - - log::trace!("assert block {block}"); - - // Check block meta, excluding fast-sync-derived boundary fields - // that can differ in the restored chain prefix. - let alice_meta = alice.db.block_meta(block); - let bob_meta = bob.db.block_meta(block); - assert!( - alice_meta.prepared && bob_meta.prepared, - "Block {block} is not prepared for Alice or Bob" - ); - assert_eq!( - alice_meta.latest_era_validators_committed, - bob_meta.latest_era_validators_committed - ); - - if let Some(mb_hash) = alice_meta - .last_committed_mb - .filter(|mb_hash| Some(*mb_hash) == bob_meta.last_committed_mb) - && alice.db.mb_compact_block(mb_hash).is_some() - && bob.db.mb_compact_block(mb_hash).is_some() - && alice.db.mb_meta(mb_hash).computed == bob.db.mb_meta(mb_hash).computed - { - assert_eq!( - alice.db.mb_program_states(mb_hash), - bob.db.mb_program_states(mb_hash) - ); - assert_eq!(alice.db.mb_schedule(mb_hash), bob.db.mb_schedule(mb_hash)); - if mb_hash != fast_synced_mb { - assert_eq!(alice.db.mb_outcome(mb_hash), bob.db.mb_outcome(mb_hash)); - } - } - - assert_eq!(alice.db.block_header(block), bob.db.block_header(block)); - assert_eq!(alice.db.block_events(block), bob.db.block_events(block)); - assert_eq!(alice.db.block_synced(block), bob.db.block_synced(block)); - - let header = alice.db.block_header(block).unwrap(); - block = header.parent_hash; - } - }; - - let config = TestEnvConfig { - validators: ValidatorsConfig::PreDefined(3), - network: EnvNetworkConfig::Enabled, - ..Default::default() - }; - let mut env = TestEnv::new(config).await.unwrap(); - - test_info!("Starting validators"); - let mut validators = Vec::new(); - for (i, validator) in env.validators.clone().into_iter().enumerate() { - let mut node = env - .new_node(NodeConfig::named(format!("validator-{i}")).validator(validator)) - .await; - node.start_service().await; - validators.push(node); - } - - let observer_events = env.new_observer_events(); - - test_info!("Creating ping program and finalizing it in MB"); - let code_id = env - .upload_code(demo_ping::WASM_BINARY) - .await - .unwrap() - .wait_for() - .await - .unwrap() - .code_id; - let ping_id = env - .create_program(code_id, 500_000_000_000_000) - .await - .unwrap() - .wait_for() - .await - .unwrap() - .program_id; - let reply = env - .send_message(ping_id, b"PING") - .await - .unwrap() - .wait_for() - .await - .unwrap(); - assert_eq!(reply.payload, b"PONG"); - - let alice_mb = tokio::time::timeout(Duration::from_secs(45), async { - observer_events - .filter_map_block_synced() - .find_map(|event| match event { - BlockEvent::Router(ethexe_common::events::RouterEvent::MBCommitted( - ethexe_common::events::router::MBCommittedEvent(mb_hash), - )) => { - let meta = validators[0].db.mb_meta(mb_hash); - (meta.computed - && !meta.last_advanced_eb.is_zero() - && validators[0].db.mb_program_states(mb_hash).is_some() - && validators[0].db.mb_outcome(mb_hash).is_some() - && validators[0].db.mb_schedule(mb_hash).is_some()) - .then_some(mb_hash) - } - _ => None, - }) - .await - }) - .await - .expect("Alice did not commit an MB that advanced an Ethereum block"); - let fast_sync_target = validators[0].db.mb_meta(alice_mb).last_advanced_eb; - assert!(!fast_sync_target.is_zero()); - test_info!("Fast-sync source MB {alice_mb}, target EB {fast_sync_target}"); - - test_info!("Starting Bob with fast sync"); - let mut bob = env.new_node(NodeConfig::named("Bob").fast_sync()).await; - tokio::time::timeout(Duration::from_secs(45), bob.start_service()) - .await - .expect("Bob did not finish fast sync"); - - let fast_synced_blocks = bob.latest_fast_synced_blocks.take().unwrap(); - let fast_synced_mb = - assert_fast_sync_boundary(fast_synced_blocks.eb_hash, &validators[0], &bob); - - test_info!("Sending post-fast-sync message and comparing Bob with Alice"); - let mut alice_events = validators[0].events(); - let mut bob_events = bob.events(); - let reply = env - .send_message(ping_id, b"PING") - .await - .unwrap() - .wait_for() - .await - .unwrap(); - assert_eq!(reply.payload, b"PONG"); - - let latest_block = env.latest_block().await.hash; - wait_till_block_prepared(&mut alice_events, latest_block).await; - wait_till_block_prepared(&mut bob_events, latest_block).await; - assert_chain( - latest_block, - fast_synced_blocks.eb_hash, - fast_synced_mb, - &validators[0], - &bob, - ); - - validators.push(bob); - stop_nodes(validators).await; -} - #[tokio::test] #[ntest::timeout(120_000)] async fn re_genesis_with_state_dump() { From 341acc75fe5b8b725b509cad8157de8f4b46a052 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Sun, 14 Jun 2026 15:02:15 +0300 Subject: [PATCH 55/72] fix(ethexe): skip processed fast sync codes --- ethexe/service/src/fast_sync.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 7e330da096d..8139fd95ebe 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -516,6 +516,9 @@ async fn instrument_codes( db: &Database, mut code_ids: BTreeSet, ) -> Result<()> { + // `code_valid` is compute's processed marker; valid codes must have instrumented bytes. + code_ids.retain(|&code_id| db.code_valid(code_id).is_none()); + if code_ids.is_empty() { log::info!("No codes to instrument. Skipping..."); return Ok(()); @@ -769,8 +772,9 @@ mod tests { use super::*; use ethexe_common::{BlockHeader, SimpleBlockData}; use ethexe_ethereum::IntoBlockId; + use ethexe_processor::Processor; use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap}, ops::RangeInclusive, }; @@ -796,6 +800,19 @@ mod tests { } } + #[tokio::test] + async fn instrument_codes_skips_already_processed_codes() { + let db = Database::memory(); + let code_id = CodeId::from([1; 32]); + db.set_code_valid(code_id, true); + let processor = Processor::new(db.clone()).unwrap(); + let mut compute = ComputeService::new(db.clone(), processor); + + instrument_codes(&mut compute, &db, BTreeSet::from([code_id])) + .await + .unwrap(); + } + fn test_block(hash: H256, parent_hash: H256, events: Vec) -> BlockData { BlockData { hash, From fe5b692c35912bdbdb24fdb6c28d6427ea67e5f6 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Sun, 14 Jun 2026 15:10:30 +0300 Subject: [PATCH 56/72] fix(ethexe): avoid latest block fast sync fallback --- ethexe/service/src/fast_sync.rs | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 8139fd95ebe..ea969e8d5b5 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -631,31 +631,9 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { .context("failed to get genesis block")?; let block_loader = observer.block_loader(); - let latest_block = observer - .block_loader() - .load_simple(BlockId::latest()) - .await - .context("failed to get latest block")? - .hash; - - let mut state_query_block = finalized_block; - let event_data = match EventData::collect(&block_loader, db, finalized_block).await? { - Some(event_data) => event_data, - None if latest_block != finalized_block => { - let Some(latest_event_data) = - EventData::collect(&block_loader, db, latest_block).await? - else { - log::warn!("No committed MB found. Skipping fast synchronization..."); - return Ok(()); - }; - log::warn!("No finalized committed MB found; trying latest block candidates"); - state_query_block = latest_block; - latest_event_data - } - None => { - log::warn!("No committed MB found. Skipping fast synchronization..."); - return Ok(()); - } + let Some(event_data) = EventData::collect(&block_loader, db, finalized_block).await? else { + log::warn!("No finalized committed MB found. Skipping fast synchronization..."); + return Ok(()); }; let EventData { @@ -684,7 +662,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { ) .await?; let program_state_hashes = - collect_program_state_hashes(observer, state_query_block, &program_code_ids).await?; + collect_program_state_hashes(observer, finalized_block, &program_code_ids).await?; let program_states = sync_from_network(network, db, &code_ids, program_state_hashes).await; From 2f5dd61780b1d8e8dca2661cb0027635080861a1 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Sun, 14 Jun 2026 15:39:10 +0300 Subject: [PATCH 57/72] Simplify fast sync committed anchor --- ethexe/service/src/fast_sync.rs | 80 +++++++++++++++--------- ethexe/service/src/tests/mod.rs | 13 ++-- ethexe/service/src/tests/utils/env.rs | 13 ++-- ethexe/service/src/tests/utils/events.rs | 10 +-- 4 files changed, 64 insertions(+), 52 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index ea969e8d5b5..0da01fbce85 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -49,7 +49,7 @@ use std::{ struct EventData { latest_committed_batch: Digest, - latest_committed_chain: FastSyncReplayTarget, + replay_target: FastSyncReplayTarget, } impl EventData { @@ -59,10 +59,10 @@ impl EventData { highest_block: H256, ) -> Result> { let mut latest_committed_batch = None; - let mut latest_committed_chain = None; + let mut replay_target = None; let mut block = highest_block; - while !db.block_meta(block).prepared { + 'blocks: while !db.block_meta(block).prepared { let block_data = block_loader.load(block, None).await?; let mut pending_eb = None; @@ -77,32 +77,31 @@ impl EventData { pending_eb = Some(*eb_hash); } BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb_hash))) => { - latest_committed_chain.get_or_insert(FastSyncReplayTarget { - mb_hash: *mb_hash, - eb_hash: pending_eb.take().unwrap_or_default(), - }); + if let Some(eb_hash) = pending_eb.take() { + replay_target = Some(FastSyncReplayTarget { + mb_hash: *mb_hash, + eb_hash, + }); + break 'blocks; + } } _ => {} } } - if latest_committed_chain.is_some() { - break; - } - block = block_data.header.parent_hash; if block.is_zero() { break; } } - let Some(latest_committed_chain) = latest_committed_chain else { + let Some(replay_target) = replay_target else { return Ok(None); }; Ok(Some(Self { latest_committed_batch: latest_committed_batch.unwrap_or_default(), - latest_committed_chain, + replay_target, })) } } @@ -638,15 +637,13 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { let EventData { latest_committed_batch, - latest_committed_chain, + replay_target, } = event_data; - let latest_committed_mb = latest_committed_chain.mb_hash; - let latest_committed_eb = if latest_committed_chain.eb_hash.is_zero() { - genesis_block_hash - } else { - latest_committed_chain.eb_hash - }; + let FastSyncReplayTarget { + mb_hash: latest_committed_mb, + eb_hash: latest_committed_eb, + } = replay_target; let block_data = block_loader.load(latest_committed_eb, None).await?; let code_ids = collect_code_ids( @@ -723,7 +720,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { if let Some(malachite) = malachite.as_mut() { // `Service::run` performs fast sync before `run_inner().start_app_task()`, // so live Malachite callbacks cannot race this startup replay gate. - let _ = malachite.enable_fast_sync_replay_filter(latest_committed_chain)?; + let _ = malachite.enable_fast_sync_replay_filter(replay_target)?; malachite.receive_new_chain_head(block_data.to_simple()); } @@ -735,10 +732,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { #[cfg(test)] sender .send(crate::tests::utils::TestingEvent::FastSyncDone( - crate::tests::utils::LatestFastSyncedBlocks { - eb_hash: latest_committed_eb, - mb_hash: latest_committed_mb, - }, + replay_target, )) .await; @@ -842,12 +836,12 @@ mod tests { .unwrap() .expect("committed chain found"); - assert_eq!(event_data.latest_committed_chain.mb_hash, newer_mb); - assert_eq!(event_data.latest_committed_chain.eb_hash, newer_eb); + assert_eq!(event_data.replay_target.mb_hash, newer_mb); + assert_eq!(event_data.replay_target.eb_hash, newer_eb); } #[tokio::test] - async fn event_data_does_not_mix_newer_mb_with_older_eb() { + async fn event_data_skips_newer_mb_without_eb_anchor() { let db = Database::memory(); let older_block = H256::from_low_u64_be(1); let newer_block = H256::from_low_u64_be(2); @@ -883,7 +877,33 @@ mod tests { .unwrap() .expect("committed chain found"); - assert_eq!(event_data.latest_committed_chain.mb_hash, newer_mb); - assert_eq!(event_data.latest_committed_chain.eb_hash, H256::zero()); + assert_eq!(event_data.replay_target.mb_hash, older_mb); + assert_eq!(event_data.replay_target.eb_hash, older_eb); + } + + #[tokio::test] + async fn event_data_ignores_committed_mb_without_eb_anchor() { + let db = Database::memory(); + let block = H256::from_low_u64_be(1); + let mb = H256::repeat_byte(0x11); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + block, + test_block( + block, + H256::zero(), + vec![BlockEvent::Router(RouterEvent::MBCommitted( + MBCommittedEvent(mb), + ))], + ), + ); + + assert!( + EventData::collect(&loader, &db, block) + .await + .unwrap() + .is_none() + ); } } diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 8cfb9dbd689..53835de4fa2 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -6,9 +6,9 @@ pub(crate) mod utils; use crate::tests::utils::{ - EnvNetworkConfig, GenesisInitializerFromDump, InfiniteStreamExt, LatestFastSyncedBlocks, Node, - NodeConfig, TestEnv, TestEnvConfig, TestingEvent, TestingEventReceiver, TestingNetworkEvent, - TestingRpcEvent, ValidatorsConfig, init_logger, stop_nodes, test_info, + EnvNetworkConfig, GenesisInitializerFromDump, InfiniteStreamExt, Node, NodeConfig, TestEnv, + TestEnvConfig, TestingEvent, TestingEventReceiver, TestingNetworkEvent, TestingRpcEvent, + ValidatorsConfig, init_logger, stop_nodes, test_info, }; use alloy::{ primitives::U256, @@ -33,6 +33,7 @@ use ethexe_common::{ use ethexe_consensus::BatchCommitter; use ethexe_db::{Database, dump::StateDump, verifier::IntegrityVerifier}; use ethexe_ethereum::{EthereumBuilder, TryGetReceipt, router::Router}; +use ethexe_malachite::FastSyncReplayTarget; use ethexe_processor::Processor; use ethexe_rpc::InjectedClient; use ethexe_runtime_common::{RUNTIME_ID, state::Storage}; @@ -3148,11 +3149,11 @@ async fn reply_callback() { async fn fast_sync() { init_logger(); - let assert_chain = |latest_block, fast_synced_blocks, alice: &Node, bob: &Node| { - let LatestFastSyncedBlocks { + let assert_chain = |latest_block, replay_target, alice: &Node, bob: &Node| { + let FastSyncReplayTarget { eb_hash: fast_synced_eb, mb_hash: fast_synced_mb, - } = fast_synced_blocks; + } = replay_target; log::info!("Assert chain in range {latest_block}..{fast_synced_eb}"); diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index f9ac1d81f9d..ea5fa29fe0d 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -6,10 +6,7 @@ use crate::{ config::EthereumConfig, tests::utils::{ InfiniteStreamExt, TestingEvent, TestingNetworkEvent, - events::{ - self, LatestFastSyncedBlocks, ObserverEventReceiver, ObserverEventSender, - TestingEventReceiver, - }, + events::{self, ObserverEventReceiver, ObserverEventSender, TestingEventReceiver}, }, }; use alloy::{ @@ -42,9 +39,9 @@ use ethexe_ethereum::{ router::RouterQuery, }; use ethexe_malachite::{ - InjectedTxMempool, MalachiteConfig, MalachiteConfigEnvironment, MalachiteService, - Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, derive_libp2p_secret, - malachite_libp2p_peer_id, + FastSyncReplayTarget, InjectedTxMempool, MalachiteConfig, MalachiteConfigEnvironment, + MalachiteService, Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, + derive_libp2p_secret, malachite_libp2p_peer_id, }; use ethexe_network::{NetworkConfig, NetworkRuntimeConfig, NetworkService, export::Multiaddr}; use ethexe_observer::{ @@ -1011,7 +1008,7 @@ pub struct Node { pub name: Option, pub db: Database, pub multiaddr: Option, - pub latest_fast_synced_blocks: Option, + pub latest_fast_synced_blocks: Option, pub custom_committer: Option>, router_query: RouterQuery, diff --git a/ethexe/service/src/tests/utils/events.rs b/ethexe/service/src/tests/utils/events.rs index 55011bbe0ad..ef63f93eedd 100644 --- a/ethexe/service/src/tests/utils/events.rs +++ b/ethexe/service/src/tests/utils/events.rs @@ -20,7 +20,7 @@ use ethexe_common::{ use ethexe_compute::ComputeEvent; use ethexe_consensus::ConsensusEvent; use ethexe_db::Database; -use ethexe_malachite::MalachiteEvent; +use ethexe_malachite::{FastSyncReplayTarget, MalachiteEvent}; use ethexe_network::{NetworkEvent, NetworkInjectedEvent, export::PeerId}; use ethexe_observer::ObserverEvent; use ethexe_rpc::RpcEvent; @@ -42,12 +42,6 @@ pub type TestingEventReceiver = KickingStream>; pub type ObserverEventSender = EventSender; pub type ObserverEventReceiver = KickingStream>; -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub struct LatestFastSyncedBlocks { - pub eb_hash: H256, - pub mb_hash: H256, -} - #[derive(Debug, Clone, Eq, PartialEq)] pub enum TestingNetworkInjectedEvent { InboundTransaction { @@ -130,7 +124,7 @@ impl TestingRpcEvent { #[derive(Debug, Clone, Eq, PartialEq)] pub enum TestingEvent { // Fast sync done. Sent just once. - FastSyncDone(LatestFastSyncedBlocks), + FastSyncDone(FastSyncReplayTarget), // Basic event to notify that service has started. Sent just once. ServiceStarted, // Services events. From b9039b7fb9dee8c62cc7cfd8ebb67875209fcbcf Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Sun, 14 Jun 2026 15:53:53 +0300 Subject: [PATCH 58/72] fix fast sync batch digest invariant --- ethexe/service/src/fast_sync.rs | 46 +++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 0da01fbce85..5d2c4be3cd2 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -100,7 +100,9 @@ impl EventData { }; Ok(Some(Self { - latest_committed_batch: latest_committed_batch.unwrap_or_default(), + // Router.commitBatch emits BatchCommitted before MB/EBCommitted; otherwise we'd seed a bogus previous batch. + latest_committed_batch: latest_committed_batch + .context("committed MB replay target without BatchCommitted event")?, replay_target, })) } @@ -742,7 +744,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use ethexe_common::{BlockHeader, SimpleBlockData}; + use ethexe_common::{BlockHeader, Digest, SimpleBlockData}; use ethexe_ethereum::IntoBlockId; use ethexe_processor::Processor; use std::{ @@ -806,6 +808,7 @@ mod tests { let older_eb = H256::repeat_byte(0x22); let newer_mb = H256::repeat_byte(0x33); let newer_eb = H256::repeat_byte(0x44); + let newer_batch = Digest::random(); let mut loader = TestBlockLoader::default(); loader.blocks.insert( @@ -816,6 +819,9 @@ mod tests { vec![ BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(newer_mb))), BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(newer_eb))), + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest: newer_batch, + })), ], ), ); @@ -838,6 +844,7 @@ mod tests { assert_eq!(event_data.replay_target.mb_hash, newer_mb); assert_eq!(event_data.replay_target.eb_hash, newer_eb); + assert_eq!(event_data.latest_committed_batch, newer_batch); } #[tokio::test] @@ -848,6 +855,7 @@ mod tests { let older_mb = H256::repeat_byte(0x11); let older_eb = H256::repeat_byte(0x22); let newer_mb = H256::repeat_byte(0x33); + let older_batch = Digest::random(); let mut loader = TestBlockLoader::default(); loader.blocks.insert( @@ -868,6 +876,9 @@ mod tests { vec![ BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(older_mb))), BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(older_eb))), + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest: older_batch, + })), ], ), ); @@ -879,6 +890,37 @@ mod tests { assert_eq!(event_data.replay_target.mb_hash, older_mb); assert_eq!(event_data.replay_target.eb_hash, older_eb); + assert_eq!(event_data.latest_committed_batch, older_batch); + } + + #[tokio::test] + async fn event_data_rejects_committed_chain_without_batch_digest() { + let db = Database::memory(); + let block = H256::from_low_u64_be(1); + let mb = H256::repeat_byte(0x11); + let eb = H256::repeat_byte(0x22); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + block, + test_block( + block, + H256::zero(), + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(eb))), + ], + ), + ); + + let Err(err) = EventData::collect(&loader, &db, block).await else { + panic!("expected missing batch digest error"); + }; + + assert!( + err.to_string() + .contains("committed MB replay target without BatchCommitted event") + ); } #[tokio::test] From 5a5a08ae9b246bd72e8a5132b2095187006e4cb2 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Sun, 14 Jun 2026 15:57:16 +0300 Subject: [PATCH 59/72] Document Malachite fast sync chain head seed --- ethexe/service/src/fast_sync.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 5d2c4be3cd2..bbf9d77c8f2 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -723,6 +723,9 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { // `Service::run` performs fast sync before `run_inner().start_app_task()`, // so live Malachite callbacks cannot race this startup replay gate. let _ = malachite.enable_fast_sync_replay_filter(replay_target)?; + // Fast sync restores DB globals directly, but Malachite keeps its + // own in-memory chain head for proposal/validation. Seed it now; the + // observer will only deliver later EBs after the app task starts. malachite.receive_new_chain_head(block_data.to_simple()); } From 607bdf6dc6af93089bb04d0d5c435458ae3a2c4e Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Tue, 16 Jun 2026 15:56:37 +0300 Subject: [PATCH 60/72] Remove stale fast-sync delay wait --- ethexe/service/src/tests/mod.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 53835de4fa2..5c59268c752 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -3336,16 +3336,6 @@ async fn fast_sync() { test_info!("📗 Starting Bob again to check how it handles partially empty database"); bob.start_service().await; - // Mine some blocks so Bob can produce the event we will wait for. - // We mine several blocks here to ensure that Bob and Alice would converge to the same chain of announces. - // Why do we need that? Because Bob was disabled he missed some announces that Alice produced, - // this announces was not committed, so Bob would not see them during fast-sync - // and would not have them in his database. This is normal situation, after a few blocks Bob and Alice should - // converge to the same chain of announces. - for _ in 0..env.commitment_delay_limit.get() { - env.skip_blocks(1).await; - } - let latest_block = env.latest_block().await.hash; let alice_latest_advanced_mb = alice.events().find_advanced_mb(latest_block).await; let bob_latest_advanced_mb = bob.events().find_advanced_mb(latest_block).await; From c07d3fe89773340dd1f94105f756afd0f15cc814 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Tue, 16 Jun 2026 19:58:00 +0300 Subject: [PATCH 61/72] Fix fast sync computed MB wait --- ethexe/service/src/tests/mod.rs | 41 ++++++------ ethexe/service/src/tests/utils/events.rs | 82 +++++++++++++++++++++++- 2 files changed, 99 insertions(+), 24 deletions(-) diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 5c59268c752..678ae240332 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -7,8 +7,8 @@ pub(crate) mod utils; use crate::tests::utils::{ EnvNetworkConfig, GenesisInitializerFromDump, InfiniteStreamExt, Node, NodeConfig, TestEnv, - TestEnvConfig, TestingEvent, TestingEventReceiver, TestingNetworkEvent, TestingRpcEvent, - ValidatorsConfig, init_logger, stop_nodes, test_info, + TestEnvConfig, TestingEvent, TestingNetworkEvent, TestingRpcEvent, ValidatorsConfig, + init_logger, stop_nodes, test_info, }; use alloy::{ primitives::U256, @@ -1480,7 +1480,7 @@ async fn reorg_deeper_than_quarantine() { let latest_block = env.latest_block().await; test_info!("Waiting for {latest_block} to be finalized in MB"); - node.events().find_advanced_mb(latest_block.hash).await; + node.events().find_mb_advanced_eb(latest_block.hash).await; test_info!("📗 Reverting Anvil to deep snapshot — past quarantine"); env.provider @@ -1498,7 +1498,7 @@ async fn reorg_deeper_than_quarantine() { let mut receiver = node.new_events(); // Here we take in account kicking stream - latest_block is not passed quarantine yet, // but kicks will generate new anvil blocks, but still this block cannot be finalized in mb, because branch is broken. - let waiting_future = receiver.find_advanced_mb(latest_block.hash); + let waiting_future = receiver.find_mb_advanced_eb(latest_block.hash); tokio::time::timeout(Duration::from_secs(20), waiting_future) .await .expect_err("block should not be finalized within 20 seconds after deep reorg"); @@ -3267,11 +3267,10 @@ async fn fast_sync() { } let latest_block = env.latest_block().await.hash; - alice.events().find_advanced_mb(latest_block).await; + alice.events().find_mb_advanced_eb(latest_block).await; test_info!("Starting Bob (fast-sync)"); let mut bob = env.new_node(NodeConfig::named("Bob").fast_sync()).await; - bob.start_service().await; test_info!("📗 Sending messages to programs"); @@ -3291,16 +3290,11 @@ async fn fast_sync() { } let latest_block = env.latest_block().await.hash; - // alice - // .events() - // .find_mb_computed() - // .find_announce_computed(latest_block) - // .await; let mut alice_events = alice.events(); let mut bob_events = bob.events(); - let (alice_mb_hash, bob_mb_hash) = tokio::join!( - alice_events.find_advanced_mb(latest_block), - bob_events.find_advanced_mb(latest_block) + tokio::join!( + alice_events.find_mb_computed_eb(latest_block), + bob_events.find_mb_computed_eb(latest_block) ); test_info!("📗 Stopping Bob"); @@ -3331,15 +3325,18 @@ async fn fast_sync() { env.skip_blocks(100).await; let latest_block = env.latest_block().await.hash; - alice.events().find_advanced_mb(latest_block).await; + alice.events().find_mb_advanced_eb(latest_block).await; test_info!("📗 Starting Bob again to check how it handles partially empty database"); bob.start_service().await; let latest_block = env.latest_block().await.hash; - let alice_latest_advanced_mb = alice.events().find_advanced_mb(latest_block).await; - let bob_latest_advanced_mb = bob.events().find_advanced_mb(latest_block).await; - assert_eq!(alice_latest_advanced_mb, bob_latest_advanced_mb); + let mut alice_events = alice.events(); + let mut bob_events = bob.events(); + tokio::join!( + alice_events.find_mb_computed_eb(latest_block), + bob_events.find_mb_computed_eb(latest_block) + ); assert_chain( latest_block, @@ -3395,7 +3392,7 @@ async fn re_genesis_with_state_dump() { // Ensure the ping-pong state is committed on-chain before re-genesis, so the // dump taken at the new genesis block carries it. let latest_block = env.latest_block().await.hash; - node.events().find_advanced_mb(latest_block).await; + node.events().find_mb_advanced_eb(latest_block).await; log::info!( "📗 Phase 2: re-genesis the router via reinitialize + lookupGenesisHash. \ @@ -3417,7 +3414,7 @@ async fn re_genesis_with_state_dump() { // Wait until the node commits an MB covering the new genesis block before // dumping from its DB. - node.events().find_advanced_mb(new_genesis_hash).await; + node.events().find_mb_advanced_eb(new_genesis_hash).await; log::info!("📗 Phase 3: collect state dump at the new genesis block."); let dump = StateDump::collect_from_storage(&node.db, new_genesis_hash).unwrap(); @@ -3565,7 +3562,7 @@ async fn re_genesis_delayed_message() { // Ensure the delayed-send state is committed before re-genesis. let latest_block = env.latest_block().await.hash; - node.events().find_advanced_mb(latest_block).await; + node.events().find_mb_advanced_eb(latest_block).await; // Phase 2: re-genesis via reinitialize + lookupGenesisHash; the new genesis // is the block where the reinitialize tx was mined. @@ -3586,7 +3583,7 @@ async fn re_genesis_delayed_message() { // Wait until the node commits an MB covering the new genesis block before // dumping from its DB. - node.events().find_advanced_mb(new_genesis_hash).await; + node.events().find_mb_advanced_eb(new_genesis_hash).await; // Phase 3: collect dump at the new genesis block; it should still carry the // pending delayed send in the dispatch stash because the 5-block delay diff --git a/ethexe/service/src/tests/utils/events.rs b/ethexe/service/src/tests/utils/events.rs index ef63f93eedd..84e749407ea 100644 --- a/ethexe/service/src/tests/utils/events.rs +++ b/ethexe/service/src/tests/utils/events.rs @@ -347,8 +347,7 @@ impl TestingEventReceiver { /// `AdvanceTillEthereumBlock` transaction — it suffices that it is an /// ancestor of this MB's `last_advanced_eb` (i.e., it sits inside /// the eth-chain segment this MB advanced over). - #[allow(dead_code)] - pub async fn find_advanced_mb(&mut self, eb_hash: H256) -> H256 { + pub async fn find_mb_advanced_eb(&mut self, eb_hash: H256) -> H256 { self.find_map_with_db(|db, event| { let TestingEvent::Malachite(MalachiteEvent::BlockFinalized { mb_hash, .. }) = event else { @@ -382,6 +381,25 @@ impl TestingEventReceiver { .await } + /// Wait until an MB covering `eb_hash` is finalized and computed, + /// returning that MB hash. + /// + /// The finalized event identifies which MB advanced over the requested EB; + /// computation may already have happened by then, so this also accepts the + /// persisted `computed` marker instead of requiring a later event. + pub async fn find_mb_computed_eb(&mut self, eb_hash: H256) -> H256 { + let mb_hash = self.find_mb_advanced_eb(eb_hash).await; + + // `MbComputed` can be emitted before the matching finalization event + // that lets us map `eb_hash` to `mb_hash`; don't wait for a duplicate + // event after compute already persisted its marker. + if self.db().mb_meta(mb_hash).computed { + return mb_hash; + } + + self.find_mb_computed(mb_hash).await + } + pub async fn find_map_with_db( &mut self, mut f: impl FnMut(Database, TestingEvent) -> Option, @@ -443,3 +461,63 @@ impl ObserverEventReceiver { KickingStream::new(stream, kicks) } } + +#[cfg(test)] +mod tests { + use super::*; + use ethexe_common::{ + BlockHeader, + db::{CompactMb, OnChainStorageRW}, + }; + use ethexe_db::Database; + use ethexe_malachite::CommitCertificate; + + #[tokio::test] + async fn find_mb_computed_eb_returns_when_mb_was_already_computed() { + let db = Database::memory(); + let eb_hash = H256::from_low_u64_be(1); + let mb_hash = H256::from_low_u64_be(2); + + db.set_block_header( + eb_hash, + BlockHeader { + height: 1, + timestamp: 12, + parent_hash: H256::zero(), + }, + ); + db.set_mb_compact_block( + mb_hash, + CompactMb { + parent: H256::zero(), + height: 1, + operations_hash: H256::zero(), + }, + ); + db.mutate_mb_meta(mb_hash, |meta| { + meta.computed = true; + meta.last_advanced_eb = eb_hash; + }); + + let (sender, mut receiver) = channel(db, None); + sender + .send(TestingEvent::Malachite(MalachiteEvent::BlockFinalized { + cert: CommitCertificate { + height: 1, + mb_hash, + signatures: Vec::new(), + }, + height: 1, + mb_hash, + })) + .await; + + let found = receiver.find_mb_computed_eb(eb_hash); + futures::pin_mut!(found); + let found = futures::future::poll_immediate(&mut found) + .await + .expect("already-computed MB should not wait for another MbComputed event"); + + assert_eq!(found, mb_hash); + } +} From 54b300f1b8ca2da70462c344a36896fc5fbd3e21 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Tue, 16 Jun 2026 20:25:53 +0300 Subject: [PATCH 62/72] chore: fix CI checks --- Cargo.lock | 3 +++ ethexe/network/src/bitswap.rs | 17 +---------------- utils/gear-workspace-hack/Cargo.toml | 10 ++++++++-- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25ec667a3be..6ed5eba094d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7980,6 +7980,7 @@ dependencies = [ name = "gear-workspace-hack" version = "0.1.0" dependencies = [ + "aead", "ahash", "alloy", "alloy-chains", @@ -8024,10 +8025,12 @@ dependencies = [ "byteorder", "bytes", "cc", + "cipher 0.4.4", "clap 4.5.54", "clap_builder", "concurrent-queue", "const-hex", + "core2", "cranelift-bitset", "cranelift-codegen", "cranelift-codegen-meta", diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs index 9da27c708b4..d24a1d076bb 100644 --- a/ethexe/network/src/bitswap.rs +++ b/ethexe/network/src/bitswap.rs @@ -1,20 +1,5 @@ -// This file is part of Gear. -// -// Copyright (C) 2026 Gear Technologies Inc. +// Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . use beetswap::multihasher::{Multihasher, MultihasherError}; use blockstore::{block::CidError, cond_send::CondSend}; diff --git a/utils/gear-workspace-hack/Cargo.toml b/utils/gear-workspace-hack/Cargo.toml index 122d48cb376..4b6ab913953 100644 --- a/utils/gear-workspace-hack/Cargo.toml +++ b/utils/gear-workspace-hack/Cargo.toml @@ -210,6 +210,7 @@ features = ["rocksdb", "test-helpers"] ### BEGIN HAKARI SECTION [target.'cfg(not(any(target_arch = "wasm32", loom)))'.dependencies] +aead = { version = "0.5", default-features = false, features = ["getrandom", "std"] } ahash = { version = "0.8" } alloy = { version = "2", features = ["kzg", "node-bindings", "provider-anvil-api", "provider-ws", "rpc-types-beacon", "rpc-types-eth", "signer-mnemonic"] } alloy-chains = { version = "0.2" } @@ -250,10 +251,12 @@ byte-slice-cast = { version = "1", default-features = false, features = ["std"] bytemuck = { version = "1", default-features = false, features = ["const_zeroed", "derive", "min_const_generics", "must_cast"] } byteorder = { version = "1", features = ["i128"] } bytes = { version = "1", features = ["serde"] } +cipher = { version = "0.4", default-features = false, features = ["zeroize"] } clap = { version = "4", features = ["derive", "env", "string", "wrap_help"] } clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "string", "suggestions", "usage", "wrap_help"] } concurrent-queue = { version = "2" } const-hex = { version = "1", features = ["core-error", "serde"] } +core2 = { git = "https://github.com/bbqsrc/core2", rev = "545e84bcb0f235b12e21351e0c69767958efe2a7", default-features = false, features = ["std"] } cranelift-bitset = { version = "0.131", default-features = false, features = ["enable-serde"] } cranelift-codegen = { version = "0.131", default-features = false, features = ["host-arch", "pulley", "std", "timing", "unwind"] } crc32fast = { version = "1" } @@ -304,7 +307,7 @@ hash-db = { version = "0.16" } hash256-std-hasher = { version = "0.15", default-features = false, features = ["std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] } hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13", features = ["raw"] } -hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17", default-features = false, features = ["default-hasher"] } +hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17", default-features = false, features = ["allocator-api2", "default-hasher", "inline-more"] } hashbrown-986da7b5efc2b80e = { package = "hashbrown", version = "0.16", features = ["serde"] } hex = { version = "0.4", features = ["serde"] } hkdf = { version = "0.12", default-features = false, features = ["std"] } @@ -484,6 +487,7 @@ x25519-dalek = { version = "2", features = ["static_secrets"] } zeroize = { version = "1", features = ["derive", "std"] } [target.'cfg(not(any(target_arch = "wasm32", loom)))'.build-dependencies] +aead = { version = "0.5", default-features = false, features = ["getrandom", "std"] } ahash = { version = "0.8" } alloy = { version = "2", features = ["kzg", "node-bindings", "provider-anvil-api", "provider-ws", "rpc-types-beacon", "rpc-types-eth", "signer-mnemonic"] } alloy-chains = { version = "0.2" } @@ -528,10 +532,12 @@ bytemuck = { version = "1", default-features = false, features = ["const_zeroed" byteorder = { version = "1", features = ["i128"] } bytes = { version = "1", features = ["serde"] } cc = { version = "1", default-features = false, features = ["parallel"] } +cipher = { version = "0.4", default-features = false, features = ["zeroize"] } clap = { version = "4", features = ["derive", "env", "string", "wrap_help"] } clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "string", "suggestions", "usage", "wrap_help"] } concurrent-queue = { version = "2" } const-hex = { version = "1", features = ["core-error", "serde"] } +core2 = { git = "https://github.com/bbqsrc/core2", rev = "545e84bcb0f235b12e21351e0c69767958efe2a7", default-features = false, features = ["std"] } cranelift-bitset = { version = "0.131", default-features = false, features = ["enable-serde"] } cranelift-codegen = { version = "0.131", default-features = false, features = ["host-arch", "pulley", "std", "timing", "unwind"] } cranelift-codegen-meta = { version = "0.131", default-features = false, features = ["pulley"] } @@ -586,7 +592,7 @@ hash-db = { version = "0.16" } hash256-std-hasher = { version = "0.15", default-features = false, features = ["std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] } hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13", features = ["raw"] } -hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17", default-features = false, features = ["default-hasher"] } +hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17", default-features = false, features = ["allocator-api2", "default-hasher", "inline-more"] } hashbrown-986da7b5efc2b80e = { package = "hashbrown", version = "0.16", features = ["serde"] } hex = { version = "0.4", features = ["serde"] } hkdf = { version = "0.12", default-features = false, features = ["std"] } From d7e66c1101a897fc6635fa39ac6c16c78bcecfe0 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 17 Jun 2026 02:10:41 +0300 Subject: [PATCH 63/72] test(ethexe): wait for referenced MB computations --- ethexe/service/src/tests/mod.rs | 22 +++--- ethexe/service/src/tests/utils/events.rs | 91 +++++++++++++++++++++++- 2 files changed, 100 insertions(+), 13 deletions(-) diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 53e5b6a97b3..9de2e990f80 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -3298,14 +3298,13 @@ async fn fast_sync() { ); test_info!("📗 Stopping Bob"); + let replay_target = bob.latest_fast_synced_blocks.unwrap(); + bob_events + .wait_referenced_mbs_computed(latest_block, replay_target) + .await; bob.stop_service().await; - assert_chain( - latest_block, - bob.latest_fast_synced_blocks.unwrap(), - &alice, - &bob, - ); + assert_chain(latest_block, replay_target, &alice, &bob); for (i, program_id) in program_ids.into_iter().enumerate() { let i = (i * 3) as u64; @@ -3338,12 +3337,11 @@ async fn fast_sync() { bob_events.find_mb_computed_eb(latest_block) ); - assert_chain( - latest_block, - bob.latest_fast_synced_blocks.unwrap(), - &alice, - &bob, - ); + let replay_target = bob.latest_fast_synced_blocks.unwrap(); + bob_events + .wait_referenced_mbs_computed(latest_block, replay_target) + .await; + assert_chain(latest_block, replay_target, &alice, &bob); } #[tokio::test] diff --git a/ethexe/service/src/tests/utils/events.rs b/ethexe/service/src/tests/utils/events.rs index 84e749407ea..d616180db31 100644 --- a/ethexe/service/src/tests/utils/events.rs +++ b/ethexe/service/src/tests/utils/events.rs @@ -31,6 +31,7 @@ use futures::{ }; use gprimitives::H256; use std::{ + collections::HashSet, iter, pin::Pin, task::{Context, Poll, ready}, @@ -400,6 +401,28 @@ impl TestingEventReceiver { self.find_mb_computed(mb_hash).await } + pub async fn wait_referenced_mbs_computed( + &mut self, + latest_block: H256, + replay_target: FastSyncReplayTarget, + ) { + let mut pending = referenced_committed_mbs( + self.db(), + latest_block, + replay_target.eb_hash, + replay_target.mb_hash, + ) + .into_iter() + .filter(|mb_hash| !self.db().mb_meta(*mb_hash).computed) + .collect::>(); + + while !pending.is_empty() { + self.find(|event| matches!(event, TestingEvent::Compute(ComputeEvent::MbComputed(_)))) + .await; + pending.retain(|mb_hash| !self.db().mb_meta(*mb_hash).computed); + } + } + pub async fn find_map_with_db( &mut self, mut f: impl FnMut(Database, TestingEvent) -> Option, @@ -410,6 +433,35 @@ impl TestingEventReceiver { } } +fn referenced_committed_mbs( + db: &Database, + latest_block: H256, + fast_synced_eb: H256, + fast_synced_mb: H256, +) -> Vec { + let mut eb_hash = latest_block; + let mut seen = HashSet::new(); + let mut mbs = Vec::new(); + + while eb_hash != fast_synced_eb { + let meta = db.block_meta(eb_hash); + let mb_hash = meta + .last_committed_mb + .unwrap_or_else(|| panic!("block {eb_hash} must have last_committed_mb")); + + if mb_hash != fast_synced_mb && seen.insert(mb_hash) { + mbs.push(mb_hash); + } + + eb_hash = db + .block_header(eb_hash) + .unwrap_or_else(|| panic!("block {eb_hash} must have header")) + .parent_hash; + } + + mbs +} + impl ObserverEventReceiver { pub fn filter_map_block(mut self) -> KickingStream> { let kicks = self.take_kicks(); @@ -467,11 +519,48 @@ mod tests { use super::*; use ethexe_common::{ BlockHeader, - db::{CompactMb, OnChainStorageRW}, + db::{BlockMetaStorageRW, CompactMb, OnChainStorageRW}, }; use ethexe_db::Database; use ethexe_malachite::CommitCertificate; + #[test] + fn referenced_committed_mbs_collects_unique_non_anchor_mbs_top_down() { + let db = Database::memory(); + let fast_synced_eb = H256::from_low_u64_be(1); + let mid_eb = H256::from_low_u64_be(2); + let latest_eb = H256::from_low_u64_be(3); + let fast_synced_mb = H256::repeat_byte(0x11); + let first_mb = H256::repeat_byte(0x22); + let second_mb = H256::repeat_byte(0x33); + + for (hash, parent_hash) in [ + (fast_synced_eb, H256::zero()), + (mid_eb, fast_synced_eb), + (latest_eb, mid_eb), + ] { + db.set_block_header( + hash, + BlockHeader { + height: hash.to_low_u64_be() as u32, + timestamp: 0, + parent_hash, + }, + ); + } + + db.mutate_block_meta(latest_eb, |meta| meta.last_committed_mb = Some(first_mb)); + db.mutate_block_meta(mid_eb, |meta| meta.last_committed_mb = Some(second_mb)); + db.mutate_block_meta(fast_synced_eb, |meta| { + meta.last_committed_mb = Some(fast_synced_mb); + }); + + assert_eq!( + referenced_committed_mbs(&db, latest_eb, fast_synced_eb, fast_synced_mb), + vec![first_mb, second_mb] + ); + } + #[tokio::test] async fn find_mb_computed_eb_returns_when_mb_was_already_computed() { let db = Database::memory(); From 0ad87e9ab74fab4f77bb0900d6094e258c8111ce Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 17 Jun 2026 02:18:01 +0300 Subject: [PATCH 64/72] Fix docs --- ethexe/observer/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/ethexe/observer/src/lib.rs b/ethexe/observer/src/lib.rs index 30d3e1c2857..37638904e47 100644 --- a/ethexe/observer/src/lib.rs +++ b/ethexe/observer/src/lib.rs @@ -22,7 +22,6 @@ //! - [`ObserverEvent`] — Stream item: `Block` on a new head, `BlockSynced` after back-fill //! - [`SyncError`] — Error classifier: `RpcError` (recoverable, skipped) vs `Fatal` (propagated) //! - [`utils::BlockLoader`] — Trait abstracting block-data loading from Ethereum -//! - [`utils::BlockId`] — Block selector for `BlockLoader::load_simple`: `Hash(H256)`, `Latest`, `Finalized` //! - [`utils::EthereumBlockLoader`] — alloy-backed [`utils::BlockLoader`] impl //! //! ## Invariants From cc41e9e5fbec77ac08a0578a47354485931a8955 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 17 Jun 2026 14:42:04 +0300 Subject: [PATCH 65/72] Fix fast sync test race --- ethexe/service/src/tests/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 9de2e990f80..a39952c94e8 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -3222,6 +3222,11 @@ async fn fast_sync() { let config = TestEnvConfig { network: EnvNetworkConfig::Enabled, + // Keep proposals one EB behind the freshest head. On loaded CI, + // Bob can receive Alice's proposal before its Malachite chain-head + // register observes the just-mined EB. + post_quarantine_delay: 1, + kicking_per_blocks: Some(1), ..Default::default() }; let mut env = TestEnv::new(config).await.unwrap(); From adc4e25bd05097ee2b12ed1d9380751c6084eec0 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 17 Jun 2026 16:29:00 +0300 Subject: [PATCH 66/72] Remove duplicate size check in Bitswap --- ethexe/network/src/bitswap.rs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs index d24a1d076bb..5900f9d1e0c 100644 --- a/ethexe/network/src/bitswap.rs +++ b/ethexe/network/src/bitswap.rs @@ -140,18 +140,7 @@ impl blockstore::Blockstore for Blockstore { task::spawn_blocking(move || { let hash = Self::convert_multihash(&hash)?; let data = match codec { - RAW_CODEC => { - let data = db.read_by_hash(hash); - - if let Some(data) = &data - && data.len() as u64 > Self::MAX_BLOCK_SIZE - { - log::warn!("{hash} is too large: {} bytes", data.len()); - return Err(blockstore::Error::ValueTooLarge); - } - - data - } + RAW_CODEC => db.read_by_hash(hash), codec => Err(blockstore::Error::CidError(CidError::InvalidCidCodec( codec, )))?, From 3e0683a9b72ad98f5df1dae966f584fe50303f23 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 17 Jun 2026 17:01:59 +0300 Subject: [PATCH 67/72] fix(vara.eth): drain bitswap inner events --- ethexe/network/src/bitswap.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs index 5900f9d1e0c..76b8bc9b185 100644 --- a/ethexe/network/src/bitswap.rs +++ b/ethexe/network/src/bitswap.rs @@ -346,21 +346,19 @@ impl NetworkBehaviour for Behaviour { self.requests.insert(query_id, channel); } - if let Poll::Ready(to_swarm) = self.inner.poll(cx) { - return match to_swarm { - ToSwarm::GenerateEvent(event) => { + loop { + match self.inner.poll(cx) { + Poll::Ready(ToSwarm::GenerateEvent(event)) => { self.handle_inner_event(event); - Poll::Pending } - to_swarm => { - Poll::Ready(to_swarm.map_out(|_event| { + Poll::Ready(to_swarm) => { + return Poll::Ready(to_swarm.map_out(|_event| { unreachable!("`ToSwarm::GenerateEvent` is handled above") - })) + })); } - }; + Poll::Pending => return Poll::Pending, + } } - - Poll::Pending } } From 6d25f41bccf24171ee7727422502bf38cda4fdb6 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 17 Jun 2026 17:07:24 +0300 Subject: [PATCH 68/72] Remove redundant fast sync compute wait --- ethexe/service/src/tests/mod.rs | 8 +-- ethexe/service/src/tests/utils/events.rs | 91 +----------------------- 2 files changed, 2 insertions(+), 97 deletions(-) diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index a39952c94e8..bdccd9e4102 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -3303,12 +3303,9 @@ async fn fast_sync() { ); test_info!("📗 Stopping Bob"); - let replay_target = bob.latest_fast_synced_blocks.unwrap(); - bob_events - .wait_referenced_mbs_computed(latest_block, replay_target) - .await; bob.stop_service().await; + let replay_target = bob.latest_fast_synced_blocks.unwrap(); assert_chain(latest_block, replay_target, &alice, &bob); for (i, program_id) in program_ids.into_iter().enumerate() { @@ -3343,9 +3340,6 @@ async fn fast_sync() { ); let replay_target = bob.latest_fast_synced_blocks.unwrap(); - bob_events - .wait_referenced_mbs_computed(latest_block, replay_target) - .await; assert_chain(latest_block, replay_target, &alice, &bob); } diff --git a/ethexe/service/src/tests/utils/events.rs b/ethexe/service/src/tests/utils/events.rs index d616180db31..84e749407ea 100644 --- a/ethexe/service/src/tests/utils/events.rs +++ b/ethexe/service/src/tests/utils/events.rs @@ -31,7 +31,6 @@ use futures::{ }; use gprimitives::H256; use std::{ - collections::HashSet, iter, pin::Pin, task::{Context, Poll, ready}, @@ -401,28 +400,6 @@ impl TestingEventReceiver { self.find_mb_computed(mb_hash).await } - pub async fn wait_referenced_mbs_computed( - &mut self, - latest_block: H256, - replay_target: FastSyncReplayTarget, - ) { - let mut pending = referenced_committed_mbs( - self.db(), - latest_block, - replay_target.eb_hash, - replay_target.mb_hash, - ) - .into_iter() - .filter(|mb_hash| !self.db().mb_meta(*mb_hash).computed) - .collect::>(); - - while !pending.is_empty() { - self.find(|event| matches!(event, TestingEvent::Compute(ComputeEvent::MbComputed(_)))) - .await; - pending.retain(|mb_hash| !self.db().mb_meta(*mb_hash).computed); - } - } - pub async fn find_map_with_db( &mut self, mut f: impl FnMut(Database, TestingEvent) -> Option, @@ -433,35 +410,6 @@ impl TestingEventReceiver { } } -fn referenced_committed_mbs( - db: &Database, - latest_block: H256, - fast_synced_eb: H256, - fast_synced_mb: H256, -) -> Vec { - let mut eb_hash = latest_block; - let mut seen = HashSet::new(); - let mut mbs = Vec::new(); - - while eb_hash != fast_synced_eb { - let meta = db.block_meta(eb_hash); - let mb_hash = meta - .last_committed_mb - .unwrap_or_else(|| panic!("block {eb_hash} must have last_committed_mb")); - - if mb_hash != fast_synced_mb && seen.insert(mb_hash) { - mbs.push(mb_hash); - } - - eb_hash = db - .block_header(eb_hash) - .unwrap_or_else(|| panic!("block {eb_hash} must have header")) - .parent_hash; - } - - mbs -} - impl ObserverEventReceiver { pub fn filter_map_block(mut self) -> KickingStream> { let kicks = self.take_kicks(); @@ -519,48 +467,11 @@ mod tests { use super::*; use ethexe_common::{ BlockHeader, - db::{BlockMetaStorageRW, CompactMb, OnChainStorageRW}, + db::{CompactMb, OnChainStorageRW}, }; use ethexe_db::Database; use ethexe_malachite::CommitCertificate; - #[test] - fn referenced_committed_mbs_collects_unique_non_anchor_mbs_top_down() { - let db = Database::memory(); - let fast_synced_eb = H256::from_low_u64_be(1); - let mid_eb = H256::from_low_u64_be(2); - let latest_eb = H256::from_low_u64_be(3); - let fast_synced_mb = H256::repeat_byte(0x11); - let first_mb = H256::repeat_byte(0x22); - let second_mb = H256::repeat_byte(0x33); - - for (hash, parent_hash) in [ - (fast_synced_eb, H256::zero()), - (mid_eb, fast_synced_eb), - (latest_eb, mid_eb), - ] { - db.set_block_header( - hash, - BlockHeader { - height: hash.to_low_u64_be() as u32, - timestamp: 0, - parent_hash, - }, - ); - } - - db.mutate_block_meta(latest_eb, |meta| meta.last_committed_mb = Some(first_mb)); - db.mutate_block_meta(mid_eb, |meta| meta.last_committed_mb = Some(second_mb)); - db.mutate_block_meta(fast_synced_eb, |meta| { - meta.last_committed_mb = Some(fast_synced_mb); - }); - - assert_eq!( - referenced_committed_mbs(&db, latest_eb, fast_synced_eb, fast_synced_mb), - vec![first_mb, second_mb] - ); - } - #[tokio::test] async fn find_mb_computed_eb_returns_when_mb_was_already_computed() { let db = Database::memory(); From f0afc8e22312899e79e3bcef5383719f77272383 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 17 Jun 2026 17:29:06 +0300 Subject: [PATCH 69/72] Skip zero program states --- ethexe/service/src/fast_sync.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index bbf9d77c8f2..33378456395 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -5,7 +5,7 @@ use crate::Service; use alloy::eips::BlockId; -use anyhow::{Context, Result, ensure}; +use anyhow::{Context, Result}; use ethexe_common::{ Address, BlockData, CodeAndIdUnchecked, Digest, ProgramStates, StateHashWithQueueSize, db::{ @@ -143,10 +143,12 @@ async fn collect_program_state_hashes( format!("failed to get state hash for actor {actor_id} at block {block_hash}") })?; - ensure!( - !state_hash.is_zero(), - "state hash is zero for actor {actor_id} at block {block_hash}" - ); + if state_hash.is_zero() { + log::trace!( + "skipping program with zero mirror state during fast sync: actor {actor_id}, block {block_hash}" + ); + continue; + } program_states.insert(actor_id, state_hash); } @@ -224,7 +226,7 @@ impl RequestManager { } fn add(&mut self, hash: H256, metadata: RequestMetadata) { - debug_assert_ne!( + assert_ne!( hash, H256::zero(), "zero hash is cannot be requested from db or network" From 040c734bdc6d0dd1ced1899efe7cedd96829fb87 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Wed, 17 Jun 2026 17:50:13 +0300 Subject: [PATCH 70/72] Bind fast sync batch digest to replay block --- ethexe/service/src/fast_sync.rs | 123 ++++++++++++++++++++++++++------ 1 file changed, 102 insertions(+), 21 deletions(-) diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 33378456395..34fe9c4992d 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -58,13 +58,11 @@ impl EventData { db: &Database, highest_block: H256, ) -> Result> { - let mut latest_committed_batch = None; - let mut replay_target = None; - let mut block = highest_block; - 'blocks: while !db.block_meta(block).prepared { + while !db.block_meta(block).prepared { let block_data = block_loader.load(block, None).await?; + let mut latest_committed_batch = None; let mut pending_eb = None; for event in block_data.events.iter().rev() { match event { @@ -78,11 +76,19 @@ impl EventData { } BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb_hash))) => { if let Some(eb_hash) = pending_eb.take() { - replay_target = Some(FastSyncReplayTarget { - mb_hash: *mb_hash, - eb_hash, - }); - break 'blocks; + return Ok(Some(Self { + // Keep the latest batch digest from this same + // EB block; later batch-only commits in the + // block still advance Router's previous-batch + // state for the next submission. + latest_committed_batch: latest_committed_batch.context( + "committed MB replay target without BatchCommitted event", + )?, + replay_target: FastSyncReplayTarget { + mb_hash: *mb_hash, + eb_hash, + }, + })); } } _ => {} @@ -95,16 +101,7 @@ impl EventData { } } - let Some(replay_target) = replay_target else { - return Ok(None); - }; - - Ok(Some(Self { - // Router.commitBatch emits BatchCommitted before MB/EBCommitted; otherwise we'd seed a bogus previous batch. - latest_committed_batch: latest_committed_batch - .context("committed MB replay target without BatchCommitted event")?, - replay_target, - })) + Ok(None) } } @@ -690,8 +687,6 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { latest_era_with_committed_validators, // NOTE: there is no invariant that fast sync should recover codes queue codes_queue: Default::default(), - // TODO #4812: using `latest_committed_batch` here is not correct, - // because `latest_committed_batch` is latest for finalized block, not for `block_hash`. last_committed_batch: latest_committed_batch, last_committed_mb: latest_committed_mb, last_committed_eb: latest_committed_eb, @@ -898,6 +893,92 @@ mod tests { assert_eq!(event_data.latest_committed_batch, older_batch); } + #[tokio::test] + async fn event_data_binds_batch_digest_to_selected_replay_target() { + let db = Database::memory(); + let older_block = H256::from_low_u64_be(1); + let newer_block = H256::from_low_u64_be(2); + let older_mb = H256::repeat_byte(0x11); + let older_eb = H256::repeat_byte(0x22); + let older_batch = Digest::random(); + let newer_batch = Digest::random(); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + newer_block, + test_block( + newer_block, + older_block, + vec![BlockEvent::Router(RouterEvent::BatchCommitted( + BatchCommittedEvent { + digest: newer_batch, + }, + ))], + ), + ); + loader.blocks.insert( + older_block, + test_block( + older_block, + H256::zero(), + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(older_mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(older_eb))), + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest: older_batch, + })), + ], + ), + ); + + let event_data = EventData::collect(&loader, &db, newer_block) + .await + .unwrap() + .expect("committed chain found"); + + assert_eq!(event_data.replay_target.mb_hash, older_mb); + assert_eq!(event_data.replay_target.eb_hash, older_eb); + assert_eq!(event_data.latest_committed_batch, older_batch); + } + + #[tokio::test] + async fn event_data_keeps_latest_batch_digest_from_selected_block() { + let db = Database::memory(); + let block = H256::from_low_u64_be(1); + let mb = H256::repeat_byte(0x11); + let eb = H256::repeat_byte(0x22); + let chain_batch = Digest::random(); + let later_batch = Digest::random(); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + block, + test_block( + block, + H256::zero(), + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(eb))), + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest: chain_batch, + })), + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest: later_batch, + })), + ], + ), + ); + + let event_data = EventData::collect(&loader, &db, block) + .await + .unwrap() + .expect("committed chain found"); + + assert_eq!(event_data.replay_target.mb_hash, mb); + assert_eq!(event_data.replay_target.eb_hash, eb); + assert_eq!(event_data.latest_committed_batch, later_batch); + } + #[tokio::test] async fn event_data_rejects_committed_chain_without_batch_digest() { let db = Database::memory(); From 3bcbf1a543f497d2b3dee08179e631434c9a8099 Mon Sep 17 00:00:00 2001 From: Grigorii Sobol Date: Tue, 23 Jun 2026 14:06:49 +0200 Subject: [PATCH 71/72] refactor(vara.eth/malachite): move EB-prerequisite gating out of Malachite Replaces the in-Malachite fast-sync replay filter with a layered design, on top of `gsobol/ethexe/al-bitswap` (al/ethexe/bitswap + tune-malachite-errors). A pure app-level "wait for BlockFinalized(target)" gate is impossible against tune's externalities: `try_emit_or_queue` head-of-line-blocks the event queue on an unprepared prerequisite EB, so the target event would never reach the app. So the prerequisite gate is moved out of Malachite: - Malachite externalities no longer queue events behind a prepared-EB prerequisite (try_emit_or_queue / pending_events / drain_pending_events / prerequisite_satisfied removed; receive_eb_prepared gone). process_mb_* emit immediately and strictly in order. - ethexe-compute now owns the "defer an MB until the EB it advances to is prepared" gate (ComputeSubService::deferred + receive_prepared_block), scanning all deferred requests on each BlockPrepared since the prepare sub-service only reports the chain head; defer-at-launch avoids head-of-line blocking ready requests. - The service applies an app-level ReplayGate after fast sync: it suppresses replayed BlockProposals until the committed target MB is reached (by hash for a fresh DB, or via a one-shot ancestry walk when the target was already finalized locally before start), then resumes live consensus. A missing CompactMb mid-walk fails loudly instead of hanging. cargo fmt, cargo clippy and cargo nextest run -p "ethexe-*" all pass (543 tests, incl. the fast_sync integration test). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EgjaE68s3nARWKBCa4Jid1 --- ethexe/compute/src/compute.rs | 92 ++++++++++++---- ethexe/compute/src/service.rs | 12 +- ethexe/malachite/service/src/externalities.rs | 89 ++++----------- ethexe/malachite/service/src/lib.rs | 2 +- ethexe/malachite/service/src/service.rs | 6 - ethexe/malachite/service/src/starter.rs | 1 - ethexe/service/src/fast_sync.rs | 103 +++++++++++++++++- ethexe/service/src/lib.rs | 28 +++-- 8 files changed, 220 insertions(+), 113 deletions(-) diff --git a/ethexe/compute/src/compute.rs b/ethexe/compute/src/compute.rs index 581760cbd67..10ab28eaec0 100644 --- a/ethexe/compute/src/compute.rs +++ b/ethexe/compute/src/compute.rs @@ -13,8 +13,8 @@ use crate::{ComputeError, ComputeEvent, ProcessorExt, Result, service::SubServic use ethexe_common::{ PromiseEmissionMode, PromisePolicy, SimpleBlockData, db::{ - CodesStorageRW, CompactMb, ConfigStorageRO, GlobalsStorageRO, MbStorageRO, MbStorageRW, - OnChainStorageRO, + BlockMetaStorageRO, CodesStorageRW, CompactMb, ConfigStorageRO, GlobalsStorageRO, + MbStorageRO, MbStorageRW, OnChainStorageRO, }, events::BlockRequestEvent, injected::Promise, @@ -85,6 +85,13 @@ pub struct ComputeSubService { metrics: Metrics, input: VecDeque, + /// Requests whose prerequisite EB (the block this MB advances to) is not + /// yet prepared in the DB. Held here instead of executing — and moved back + /// into `input` by [`Self::receive_prepared_block`] once the prerequisite + /// lands. This is the gate Malachite used to apply before emitting events; + /// owning it here lets replayed/early MBs flow through as events while + /// their execution waits for the code-validation pipeline to catch up. + deferred: VecDeque, /// Head of the in-flight computation, kept so [`Self::receive_mb`] can /// skip duplicates that would otherwise re-emit `MbComputed`. in_flight_mb: Option, @@ -111,6 +118,7 @@ impl ComputeSubService

{ promise_emission_mode, metrics: Metrics::default(), input: VecDeque::new(), + deferred: VecDeque::new(), in_flight_mb: None, computation: None, promises_stream: None, @@ -119,12 +127,13 @@ impl ComputeSubService

{ } pub fn receive_mb(&mut self, mb_hash: H256, promise_policy: PromisePolicy) { - // Idempotent: skip if already computed, in flight, or queued — - // otherwise BlockProposal+BlockFinalized for the same head emit - // `MbComputed` twice. + // Idempotent: skip if already computed, in flight, or queued + // (`input` or `deferred`) — otherwise BlockProposal+BlockFinalized + // for the same head emit `MbComputed` twice. if self.db.mb_meta(mb_hash).computed || self.in_flight_mb == Some(mb_hash) || self.input.iter().any(|r| r.mb_hash == mb_hash) + || self.deferred.iter().any(|r| r.mb_hash == mb_hash) { return; } @@ -134,6 +143,32 @@ impl ComputeSubService

{ }); } + /// An Ethereum block has been prepared: requeue every deferred request + /// whose prerequisite EB is now satisfied. `PrepareSubService` prepares a + /// whole ancestor chain but only reports the head, so we re-check all + /// deferred requests rather than matching the reported hash. + pub fn receive_prepared_block(&mut self, _eb_hash: H256) { + let mut still_deferred = VecDeque::with_capacity(self.deferred.len()); + while let Some(req) = self.deferred.pop_front() { + if self.db.mb_meta(req.mb_hash).computed { + continue; + } + if self.prerequisite_ready(req.mb_hash) { + self.input.push_back(req); + } else { + still_deferred.push_back(req); + } + } + self.deferred = still_deferred; + } + + /// Whether the EB this MB advances to is prepared (or there is none). + /// Execution reads that EB's events, so it must wait until then. + fn prerequisite_ready(&self, mb_hash: H256) -> bool { + let eb = self.db.mb_meta(mb_hash).last_advanced_eb; + eb.is_zero() || self.db.block_meta(eb).prepared + } + async fn compute( db: Database, mut processor: P, @@ -444,25 +479,37 @@ impl SubService for ComputeSubService

{ type Output = ComputeEvent; fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll> { - // (1) Pick up the next request whenever no work is in flight. + // (1) Pick up the next ready request whenever no work is in flight. + // Skip already-computed requests and park ones whose prerequisite EB + // is not prepared yet into `deferred`, scanning past them so a parked + // request never head-of-line blocks a ready one behind it. if self.computation.is_none() && self.promises_stream.is_none() && self.pending_event.is_none() - && let Some(req) = self.input.pop_front() { - let (sender, receiver) = mpsc::unbounded_channel(); - self.in_flight_mb = Some(req.mb_hash); - self.promises_stream = Some(MbPromisesStream { receiver }); - self.computation = Some(future_timing::timed( - Self::compute( - self.db.clone(), - self.processor.clone(), - req, - self.promise_emission_mode, - sender, - ) - .boxed(), - )); + while let Some(req) = self.input.pop_front() { + if self.db.mb_meta(req.mb_hash).computed { + continue; + } + if !self.prerequisite_ready(req.mb_hash) { + self.deferred.push_back(req); + continue; + } + let (sender, receiver) = mpsc::unbounded_channel(); + self.in_flight_mb = Some(req.mb_hash); + self.promises_stream = Some(MbPromisesStream { receiver }); + self.computation = Some(future_timing::timed( + Self::compute( + self.db.clone(), + self.processor.clone(), + req, + self.promise_emission_mode, + sender, + ) + .boxed(), + )); + break; + } } // (2) Forward streaming promises before anything else so the @@ -558,6 +605,11 @@ mod tests { }, ); db.set_block_events(hash, &events); + // Compute now defers an MB until the EB it advances to is prepared, so + // these synthetic EBs (standing in for already-synced blocks) must carry + // the flag for the direct `ComputeSubService` tests that never pump a + // `BlockPrepared` notification. + db.mutate_block_meta(hash, |m| m.prepared = true); hash } diff --git a/ethexe/compute/src/service.rs b/ethexe/compute/src/service.rs index e79cbfac70d..6208db06921 100644 --- a/ethexe/compute/src/service.rs +++ b/ethexe/compute/src/service.rs @@ -98,6 +98,11 @@ impl Stream for ComputeService

{ }; if let Poll::Ready(result) = self.prepare_sub_service.poll_next(cx) { + // A freshly prepared EB may unblock MBs that were deferred + // waiting for it (the gate Malachite used to own). + if let Ok(crate::prepare::Event::BlockPrepared(hash)) = &result { + self.mb_compute_sub_service.receive_prepared_block(*hash); + } return Poll::Ready(Some(result.map(ComputeEvent::from))); }; @@ -135,8 +140,8 @@ mod tests { use ethexe_common::{ BlockHeader, CodeAndIdUnchecked, db::{ - BlockMetaStorageRO, CodesStorageRO, CompactMb, MbStorageRO, MbStorageRW, - OnChainStorageRW, + BlockMetaStorageRO, BlockMetaStorageRW, CodesStorageRO, CompactMb, MbStorageRO, + MbStorageRW, OnChainStorageRW, }, malachite::{Operation, Operations}, mock::{Tap, seed_genesis_zero_mb}, @@ -172,6 +177,9 @@ mod tests { }, ); db.set_block_events(eth_block_hash, &[]); + // Compute defers an MB until its advanced EB is prepared; mark it so the + // `compute_mb` test (which never pumps a `BlockPrepared`) runs immediately. + db.mutate_block_meta(eth_block_hash, |m| m.prepared = true); let operations_hash = db.set_operations(Operations::new(vec![ Operation::AdvanceTillEthereumBlock { diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index e81b174014f..0fad36767eb 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -57,8 +57,10 @@ use ethexe_db::Database; use ethexe_malachite_core::{Block, BlockPayload, Externalities, MAX_BLOCK_PAYLOAD_BYTES}; use gprimitives::H256; use parity_scale_codec::{DecodeAll, Encode}; -use std::{collections::VecDeque, sync::Arc}; -use tokio::sync::{RwLock, mpsc::UnboundedSender}; +use std::sync::Arc; +#[cfg(test)] +use tokio::sync::RwLock; +use tokio::sync::mpsc::UnboundedSender; use tracing::{debug, error, trace, warn}; /// Constant parameters for [`EthexeExternalities`]; @@ -81,23 +83,10 @@ pub(crate) struct EthexeExternalities { pub mempool: Option>, /// Reference to the latest chain head data. pub chain_head: Arc, - /// Pending service events queue. - /// Release events from here only when their prerequisite EB is prepared. - pub pending_events: RwLock>, /// Channel to poll events in MalachiteService. pub event_tx: UnboundedSender>, } -/// One outbound [`MalachiteEvent`] that can't be released until its -/// `prerequisite` Eth block is prepared in local DB. -pub(crate) struct PendingEvent { - /// Event body - pub event: MalachiteEvent, - /// Prerequisite Eth block hash - /// that must be prepared before this event can be emitted - pub prerequisite: H256, -} - #[async_trait] impl Externalities for EthexeExternalities { async fn process_mb_proposal(&self, mb_hash: H256, mb: Block) -> Result<()> { @@ -132,14 +121,15 @@ impl Externalities for EthexeExternalities { meta.last_advanced_eb = last_advanced; }); - self.try_emit_or_queue( - MalachiteEvent::BlockProposal { - height: mb.height, - mb_hash, - }, - last_advanced, - ) - .await; + // Emit immediately: the "wait until the advanced EB is prepared" + // gating now lives in ethexe-compute, which defers MB execution + // (not event emission) until the prerequisite EB is ready. Emitting + // here unconditionally keeps events strictly ordered and lets the + // service drain fast-sync replay without head-of-line stalls. + let _ = self.event_tx.send(Ok(MalachiteEvent::BlockProposal { + height: mb.height, + mb_hash, + })); Ok(()) } @@ -182,16 +172,12 @@ impl Externalities for EthexeExternalities { mb_hash, signatures: cert.signatures, }; - let last_advanced = self.db.mb_meta(mb_hash).last_advanced_eb; - self.try_emit_or_queue( - MalachiteEvent::BlockFinalized { - cert: app_cert, - height: cert.height, - mb_hash, - }, - last_advanced, - ) - .await; + // Emit immediately; see the note in `process_mb_proposal`. + let _ = self.event_tx.send(Ok(MalachiteEvent::BlockFinalized { + cert: app_cert, + height: cert.height, + mb_hash, + })); Ok(()) } @@ -470,39 +456,6 @@ impl Externalities for EthexeExternalities { } impl EthexeExternalities { - /// Check whether the prerequisite EB is prepared in local DB. - /// Zero hash is a special case that always passes. - fn prerequisite_satisfied(&self, prerequisite: H256) -> bool { - use ethexe_common::db::BlockMetaStorageRO; - prerequisite.is_zero() || self.db.block_meta(prerequisite).prepared - } - - /// Send event immediately if prerequisite is satisfied, otherwise queue it for later emission. - pub(crate) async fn try_emit_or_queue(&self, event: MalachiteEvent, prerequisite: H256) { - let mut queue = self.pending_events.write().await; - if queue.is_empty() && self.prerequisite_satisfied(prerequisite) { - // Channel receiver dropped only on shutdown — best-effort. - let _ = self.event_tx.send(Ok(event)); - } else { - queue.push_back(PendingEvent { - event, - prerequisite, - }); - } - } - - /// Check the pending events queue and release any events whose prerequisites are now satisfied. - pub(crate) async fn drain_pending_events(&self) { - let mut queue = self.pending_events.write().await; - while let Some(front) = queue.front() { - if !self.prerequisite_satisfied(front.prerequisite) { - break; - } - let entry = queue.pop_front().expect("just peeked"); - let _ = self.event_tx.send(Ok(entry.event)); - } - } - // Wait for either a new EB candidate past quarantine // or any suitable injected tx to include in the next proposal. async fn wait_for_proposable_content( @@ -640,7 +593,6 @@ mod tests { mempool: Some(Arc::new(EmptyMempool)), chain_head: make_chain_head(), event_tx, - pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 0, @@ -1055,7 +1007,6 @@ mod tests { mempool: Some(Arc::clone(&tracker) as Arc), chain_head: make_chain_head(), event_tx, - pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 0, @@ -1122,7 +1073,6 @@ mod tests { mempool: Some(mempool as Arc), chain_head: make_chain_head(), event_tx, - pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 0, @@ -1876,7 +1826,6 @@ mod tests { mempool: Some(Arc::new(EmptyMempool)), chain_head: make_chain_head(), event_tx, - pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 2, diff --git a/ethexe/malachite/service/src/lib.rs b/ethexe/malachite/service/src/lib.rs index eda28d5cb31..759ef8dd08b 100644 --- a/ethexe/malachite/service/src/lib.rs +++ b/ethexe/malachite/service/src/lib.rs @@ -28,7 +28,7 @@ //! - [`TxValidity`] (enum) — Validity verdict: `Valid`, `Duplicate`, `Outdated`, … //! //! Driver methods on [`MalachiteService`]: `receive_injected_transaction`, -//! `receive_new_eb`, `receive_eb_synced`, `receive_eb_prepared`, `shutdown`. +//! `receive_new_eb`, `receive_eb_synced`, `shutdown`. //! //! [`TxValidity`] gates inclusion: a producer drops any non-`Valid` tx when //! building an MB, and a validator rejects an entire MB that contains one. diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 87ddb73508b..485f92d4983 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -129,12 +129,6 @@ impl MalachiteService { } } - /// Handle a prepared Ethereum block: release pending events whose - /// prerequisite is now satisfied. - pub async fn receive_eb_prepared(&self, _eb_hash: H256) { - self.externalities.drain_pending_events().await - } - /// Tear down the inner consensus core, releasing its store and sockets. pub async fn shutdown(mut self) { if let Some(inner) = self.inner.take() { diff --git a/ethexe/malachite/service/src/starter.rs b/ethexe/malachite/service/src/starter.rs index e2da525813e..166be2a5769 100644 --- a/ethexe/malachite/service/src/starter.rs +++ b/ethexe/malachite/service/src/starter.rs @@ -105,7 +105,6 @@ impl MalachiteServiceStarter { }, mempool: mempool.clone(), chain_head: chain_head.clone(), - pending_events: Default::default(), event_tx, }; diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index b7dea7d494a..87951a22498 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -5,12 +5,12 @@ use crate::Service; use alloy::eips::BlockId; -use anyhow::{Context, Result}; +use anyhow::{Context, Result, anyhow}; use ethexe_common::{ Address, BlockData, CodeAndIdUnchecked, Digest, ProgramStates, StateHashWithQueueSize, db::{ BlockMetaStorageRO, CodesStorageRO, CodesStorageRW, ConfigStorageRO, GlobalsStorageRW, - MbStorageRW, OnChainStorageRW, PreparedBlockData, + MbStorageRO, MbStorageRW, OnChainStorageRW, PreparedBlockData, }, events::{ BlockEvent, RouterEvent, @@ -57,6 +57,97 @@ pub struct FastSyncReplayTarget { pub eb_hash: H256, } +/// Startup gate that suppresses Malachite's replay of the already-restored +/// chain after fast sync. +/// +/// When Malachite's core starts it value-syncs from genesis and replays every +/// historical MB as a `BlockProposal` (which is what drives compute; the +/// matching `BlockFinalized` always follows and does no compute). Fast sync +/// only restored the DB up to `latest_committed_mb`, so these replayed +/// proposals must not drive compute. Malachite emits blocks in height order, so +/// it suffices to drop proposals until the target MB is reached. Two cases: +/// * fresh DB — Malachite replays from genesis; we wait for the target by hash; +/// * target already finalized locally before this start — Malachite resumes +/// from the target's child, so the target hash never reappears. The first +/// proposal is then a descendant of the target; an ancestry walk detects that +/// and opens the gate immediately. +/// +/// The ancestry walk runs at most once (the first proposal); afterwards the +/// decision is a single hash comparison. +pub(crate) enum ReplayGate { + /// No fast sync this run — never gates. + Open, + /// Awaiting the first proposal to disambiguate the two cases above. + Pending(H256), + /// Genesis replay confirmed; waiting for the target MB by hash. + Waiting(H256), +} + +impl ReplayGate { + pub(crate) fn new(target: Option) -> Self { + match target { + Some(target) => Self::Pending(target.mb_hash), + None => Self::Open, + } + } + + /// Whether the `BlockProposal` for `mb_hash` should drive compute; opens + /// the gate once the target boundary is reached. Only proposals are gated + /// because compute is triggered solely by `BlockProposal` (the matching + /// `BlockFinalized` always follows and does no compute). + pub(crate) fn allow(&mut self, db: &Database, mb_hash: H256) -> Result { + match *self { + Self::Open => Ok(true), + Self::Pending(target) => { + if mb_hash == target { + // The restored boundary itself — already computed by fast sync. + *self = Self::Open; + Ok(false) + } else if target_is_ancestor(db, target, mb_hash)? { + // Target was finalized before this start; this is the first + // live descendant — process it and resume normally. + *self = Self::Open; + Ok(true) + } else { + *self = Self::Waiting(target); + Ok(false) + } + } + Self::Waiting(target) => { + if mb_hash == target { + *self = Self::Open; + } + Ok(false) + } + } + } +} + +/// Walk the MB parent chain from `mb_hash` and report whether `target` is a +/// strict ancestor. Errors loudly on a missing `CompactMb` rather than hanging +/// the gate on incomplete data. +fn target_is_ancestor(db: &Database, target: H256, mb_hash: H256) -> Result { + let mut current = mb_hash; + loop { + let parent = db + .mb_compact_block(current) + .ok_or_else(|| { + anyhow!( + "fast-sync replay gate: missing CompactMb for {current} while resolving \ + ancestry of finalized MB {mb_hash}" + ) + })? + .parent; + if parent == target { + return Ok(true); + } + if parent.is_zero() { + return Ok(false); + } + current = parent; + } +} + struct EventData { latest_committed_batch: Digest, replay_target: FastSyncReplayTarget, @@ -605,7 +696,7 @@ async fn latest_era_with_committed_validators( .context("failed to calculate era from validators timestamp") } -pub(crate) async fn sync(service: &mut Service) -> Result<()> { +pub(crate) async fn sync(service: &mut Service) -> Result> { let Service { observer, compute, @@ -617,7 +708,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { } = service; let Some(network) = network else { log::warn!("Network service is disabled. Skipping fast synchronization..."); - return Ok(()); + return Ok(None); }; log::info!("Fast synchronization is in progress..."); @@ -642,7 +733,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { let block_loader = observer.block_loader(); let Some(event_data) = EventData::collect(&block_loader, db, finalized_block).await? else { log::warn!("No finalized committed MB found. Skipping fast synchronization..."); - return Ok(()); + return Ok(None); }; let EventData { @@ -743,7 +834,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { )) .await; - Ok(()) + Ok(Some(replay_target)) } #[cfg(test)] diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index 1f458cc5ad5..1404c85474f 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -595,16 +595,18 @@ impl Service { } pub async fn run(mut self) -> Result<()> { - if self.fast_sync { - fast_sync::sync(&mut self).await?; - } + let replay_target = if self.fast_sync { + fast_sync::sync(&mut self).await? + } else { + None + }; - self.run_inner().await.inspect_err(|err| { + self.run_inner(replay_target).await.inspect_err(|err| { log::error!("Service finished work with error: {err:?}"); }) } - async fn run_inner(self) -> Result<()> { + async fn run_inner(self, replay_target: Option) -> Result<()> { let Service { db, mut network, @@ -648,6 +650,10 @@ impl Service { let mut network_injected_txs: HashMap<_, PendingNetworkInjectedTx> = HashMap::new(); + // Suppresses Malachite's startup replay of the chain fast sync already + // restored, up to and including the committed target MB. + let mut replay_gate = fast_sync::ReplayGate::new(replay_target); + loop { let event: Event = tokio::select! { event = compute.select_next_some() => event?.into(), @@ -712,8 +718,9 @@ impl Service { if let Some(c) = consensus.as_mut() { c.receive_prepared_block(block_hash)?; } - - malachite.receive_eb_prepared(block_hash).await; + // Compute owns the "wait for the prerequisite EB" gate + // now (see `ComputeSubService`), so Malachite no longer + // needs a prepared-block notification. } ComputeEvent::CodeProcessed(_) => { // Nothing @@ -943,6 +950,13 @@ impl Service { "Malachite: BlockProposal", ); + // Suppress replay of the chain fast sync already + // restored, up to and including the committed target MB. + if !replay_gate.allow(&db, mb_hash)? { + tracing::debug!(mb_hash = %mb_hash, "fast-sync replay gate: skipping replayed proposal"); + continue; + } + compute.compute_mb(mb_hash, ethexe_common::PromisePolicy::Enabled); } MalachiteEvent::BlockFinalized { From 22a694cc7a3740397e0c1f63159c5c2aa11aa8e2 Mon Sep 17 00:00:00 2001 From: Arsenii Lyashenko Date: Mon, 29 Jun 2026 23:46:44 +0300 Subject: [PATCH 72/72] Post-merge fixes --- ethexe/malachite/core/src/lib.rs | 5 +- ethexe/malachite/core/src/service.rs | 12 +-- ethexe/malachite/service/src/config.rs | 5 +- ethexe/malachite/service/src/externalities.rs | 91 +++++++------------ ethexe/malachite/service/src/service.rs | 28 +++++- ethexe/malachite/service/src/starter.rs | 21 ++++- .../service/tests/restart_resilience.rs | 2 - ethexe/service/src/fast_sync.rs | 18 ++-- ethexe/service/src/lib.rs | 10 +- ethexe/service/src/tests/utils/env.rs | 6 +- 10 files changed, 101 insertions(+), 97 deletions(-) diff --git a/ethexe/malachite/core/src/lib.rs b/ethexe/malachite/core/src/lib.rs index 4b0c9c15dbd..334604bf310 100644 --- a/ethexe/malachite/core/src/lib.rs +++ b/ethexe/malachite/core/src/lib.rs @@ -89,7 +89,10 @@ mod store; mod streaming; pub use crate::{ - config::{MalachiteCoreConfig, Multiaddr, NodeRole, ValidatorEntry}, + config::{ + Environment as MalachiteConfigEnvironment, MalachiteCoreConfig, Multiaddr, NodeRole, + ValidatorEntry, + }, externalities::Externalities, service::{MService, MalachiteCore}, signing::{ diff --git a/ethexe/malachite/core/src/service.rs b/ethexe/malachite/core/src/service.rs index fad791963a5..41ef6422f34 100644 --- a/ethexe/malachite/core/src/service.rs +++ b/ethexe/malachite/core/src/service.rs @@ -6,7 +6,7 @@ use crate::{ app, codec::ScaleCodec, - config::{MalachiteCoreConfig, NodeRole}, + config::{Environment, MalachiteCoreConfig, NodeRole}, context::{MalachiteCtx, Validator, ValidatorSet}, externalities::Externalities, signing::{ @@ -40,10 +40,7 @@ use std::{ task::{Context as TaskContext, Poll}, time::Duration, }; -use tokio::{ - sync::{mpsc, oneshot}, - task::JoinHandle, -}; +use tokio::{sync::mpsc, task::JoinHandle}; /// Trait-object-friendly facade for the service. The stream carries /// only fatal app-task errors — successful events reach the @@ -245,12 +242,9 @@ impl MalachiteCore { )?; // ---- spawn app task ---- - let (start_tx, start_rx) = oneshot::channel(); let (errors_tx, errors_rx) = mpsc::unbounded_channel(); let externalities_for_task = Arc::clone(&externalities); let app_handle = tokio::spawn(async move { - start_rx.await.expect("start sender has been dropped"); - if let Err(e) = app::run::(state, channels, externalities_for_task).await { tracing::error!(target: "ethexe-malachite-core", error = %e, "app task terminated"); let _ = errors_tx.send(e); @@ -258,7 +252,6 @@ impl MalachiteCore { }); Ok(Self { - start_tx: Some(start_tx), errors_rx, engine, app_handle, @@ -300,7 +293,6 @@ impl Stream for MalachiteCore { type Item = anyhow::Error; fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { - debug_assert!(self.start_tx.is_none(), "app task is not started"); self.errors_rx.poll_recv(cx) } } diff --git a/ethexe/malachite/service/src/config.rs b/ethexe/malachite/service/src/config.rs index 1a269f6af25..364808e18cd 100644 --- a/ethexe/malachite/service/src/config.rs +++ b/ethexe/malachite/service/src/config.rs @@ -8,14 +8,17 @@ //! is wired in directly — there is no separate genesis file — so the //! caller is the single source of truth for who can vote. +pub use ethexe_malachite_core::{Multiaddr, ValidatorEntry}; + use crate::Mempool; use ethexe_common::ecdsa::{PublicKey, Signer}; -pub use ethexe_malachite_core::{Multiaddr, ValidatorEntry}; +use ethexe_malachite_core::MalachiteConfigEnvironment; use std::{net::SocketAddr, path::PathBuf, time::Duration}; #[derive(Clone, Debug)] pub struct MalachiteServiceConfig { pub env: MalachiteConfigEnvironment, + /// Gas allowance per block. pub gas_allowance: u64, diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index 3aa2ee42853..ecee08f3d42 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -101,7 +101,7 @@ pub(crate) struct EthexeExternalities { /// While active it suppresses all Malachite callbacks until the /// captured target proposal and finalization are replayed. Callers /// must enable it before starting the Malachite application task. - pub fast_sync_replay_filter: Mutex>, + pub fast_sync_replay_filter: RwLock>, } /// One outbound [`MalachiteEvent`] that can't be released until its @@ -151,7 +151,10 @@ impl Externalities for EthexeExternalities { // The fast-sync replay gate sits after durable MB writes so a suppressed // replayed proposal is still available to finalization and DB walks; only // outbound events are skipped. - if self.suppress_for_fast_sync_replay_proposal(mb_hash, last_advanced)? { + if self + .suppress_for_fast_sync_replay_proposal(mb_hash, last_advanced) + .await? + { return Ok(()); } @@ -174,7 +177,10 @@ impl Externalities for EthexeExternalities { // The replay gate sits after loading the compact block and operations: // even suppressed finalizations must prove the proposal path persisted // the MB before we skip mempool, globals, and event side effects. - if self.suppress_for_fast_sync_replay_finalization(mb_hash) { + if self + .suppress_for_fast_sync_replay_finalization(mb_hash) + .await + { return Ok(()); } @@ -500,23 +506,17 @@ impl Externalities for EthexeExternalities { } impl EthexeExternalities { - pub(crate) fn enable_fast_sync_replay_filter(&self, target: FastSyncReplayTarget) { - *self - .fast_sync_replay_filter - .lock() - .expect("fast_sync_replay_filter poisoned") = + pub(crate) async fn enable_fast_sync_replay_filter(&self, target: FastSyncReplayTarget) { + *self.fast_sync_replay_filter.write().await = Some(FastSyncReplayFilter::UntilTargetProposal(target)); } - fn suppress_for_fast_sync_replay_proposal( + async fn suppress_for_fast_sync_replay_proposal( &self, mb_hash: H256, last_advanced_eb: H256, ) -> Result { - let mut guard = self - .fast_sync_replay_filter - .lock() - .expect("fast_sync_replay_filter poisoned"); + let mut guard = self.fast_sync_replay_filter.write().await; match *guard { None => Ok(false), @@ -549,11 +549,8 @@ impl EthexeExternalities { } } - fn suppress_for_fast_sync_replay_finalization(&self, mb_hash: H256) -> bool { - let mut guard = self - .fast_sync_replay_filter - .lock() - .expect("fast_sync_replay_filter poisoned"); + async fn suppress_for_fast_sync_replay_finalization(&self, mb_hash: H256) -> bool { + let mut guard = self.fast_sync_replay_filter.write().await; match *guard { None => false, @@ -745,7 +742,7 @@ mod tests { canonical_quarantine: 0, post_quarantine_delay: 0, }, - fast_sync_replay_filter: Mutex::new(None), + fast_sync_replay_filter: RwLock::new(None), }; (ext, event_rx) } @@ -1161,7 +1158,7 @@ mod tests { canonical_quarantine: 0, post_quarantine_delay: 0, }, - fast_sync_replay_filter: Mutex::new(None), + fast_sync_replay_filter: RwLock::new(None), }; let payload = Operations::new(vec![ @@ -1215,26 +1212,20 @@ mod tests { ext.enable_fast_sync_replay_filter(FastSyncReplayTarget { mb_hash: boundary_hash, eb_hash: boundary_eb, - }); + }) + .await; ext.process_mb_proposal(boundary_hash, boundary_block) .await .unwrap(); assert!(rx.try_recv().is_err(), "boundary proposal is suppressed"); assert_eq!( - *ext.fast_sync_replay_filter - .lock() - .expect("fast_sync_replay_filter poisoned"), + *ext.fast_sync_replay_filter.read().await, Some(FastSyncReplayFilter::UntilTargetFinalization { mb_hash: boundary_hash, }), ); - assert!( - ext.pending_events - .lock() - .expect("pending_events poisoned") - .is_empty() - ); + assert!(ext.pending_events.read().await.is_empty()); assert_eq!(db.mb_meta(boundary_hash).last_advanced_eb, boundary_eb); assert!(db.mb_compact_block(boundary_hash).is_some()); @@ -1245,12 +1236,7 @@ mod tests { rx.try_recv().is_err(), "boundary finalization is suppressed" ); - assert!( - ext.pending_events - .lock() - .expect("pending_events poisoned") - .is_empty() - ); + assert!(ext.pending_events.read().await.is_empty()); let post_eb = H256::repeat_byte(0xA2); db.mutate_block_meta(post_eb, |meta| meta.prepared = true); @@ -1299,7 +1285,8 @@ mod tests { ext.enable_fast_sync_replay_filter(FastSyncReplayTarget { mb_hash: boundary_hash, eb_hash: boundary_eb, - }); + }) + .await; let operations_hash = db.set_operations(boundary_payload); db.set_mb_compact_block( @@ -1319,10 +1306,7 @@ mod tests { "boundary finalization before proposal is suppressed" ); assert!( - ext.fast_sync_replay_filter - .lock() - .expect("fast_sync_replay_filter poisoned") - .is_some(), + ext.fast_sync_replay_filter.read().await.is_some(), "filter must stay active until boundary proposal is seen", ); @@ -1345,10 +1329,7 @@ mod tests { .await .unwrap(); assert!( - ext.fast_sync_replay_filter - .lock() - .expect("fast_sync_replay_filter poisoned") - .is_none(), + ext.fast_sync_replay_filter.read().await.is_none(), "boundary proposal plus finalization clears filter", ); @@ -1380,7 +1361,8 @@ mod tests { ext.enable_fast_sync_replay_filter(FastSyncReplayTarget { mb_hash, eb_hash: expected_eb, - }); + }) + .await; let err = ext .process_mb_proposal(mb_hash, block) @@ -1398,10 +1380,7 @@ mod tests { Ok(event) => panic!("expected mismatch error event, got {event:?}"), } assert!( - ext.fast_sync_replay_filter - .lock() - .expect("fast_sync_replay_filter poisoned") - .is_none(), + ext.fast_sync_replay_filter.read().await.is_none(), "mismatch must clear replay filter", ); } @@ -1418,7 +1397,8 @@ mod tests { ext.enable_fast_sync_replay_filter(FastSyncReplayTarget { mb_hash, eb_hash: expected_eb, - }); + }) + .await; let err = ext .process_mb_proposal(mb_hash, block) @@ -1436,10 +1416,7 @@ mod tests { Ok(event) => panic!("expected mismatch error event, got {event:?}"), } assert!( - ext.fast_sync_replay_filter - .lock() - .expect("fast_sync_replay_filter poisoned") - .is_none(), + ext.fast_sync_replay_filter.read().await.is_none(), "mismatch must clear replay filter", ); } @@ -1472,7 +1449,7 @@ mod tests { canonical_quarantine: 0, post_quarantine_delay: 0, }, - fast_sync_replay_filter: Mutex::new(None), + fast_sync_replay_filter: RwLock::new(None), }; (ext, event_rx) } @@ -2227,7 +2204,7 @@ mod tests { canonical_quarantine: 2, post_quarantine_delay: 3, }, - fast_sync_replay_filter: Mutex::new(None), + fast_sync_replay_filter: RwLock::new(None), }; set_head(&ext, head).await; diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 1a3f41d379d..2621479c414 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -7,12 +7,12 @@ //! engine and exposes its outputs as a `Stream` of [`MalachiteEvent`]s. use crate::{ - Mempool, ValidatorEntry, + FastSyncReplayTarget, Mempool, ValidatorEntry, externalities::EthexeExternalities, mempool::TxInsertionStatus, types::{ChainHead, MalachiteEvent}, }; -use anyhow::Result; +use anyhow::{Context as _, Result}; use ethexe_common::{ Address, SimpleBlockData, db::{ConfigStorageRO, OnChainStorageRO}, @@ -55,6 +55,30 @@ impl Drop for MalachiteService { } impl MalachiteService { + pub async fn enable_fast_sync_replay_filter( + &self, + target: FastSyncReplayTarget, + ) -> Result { + if self + .inner + .as_ref() + .context("Malachite inner service is shut down")? + .is_finalized(target.mb_hash)? + { + tracing::info!( + mb_hash = %target.mb_hash, + eb_hash = %target.eb_hash, + "not enabling fast-sync replay filter for already-finalized MB", + ); + return Ok(false); + } + + self.externalities + .enable_fast_sync_replay_filter(target) + .await; + Ok(true) + } + /// Route an injected transaction into the mempool. /// Rejects with `PoolFull` when the node is not a validator. pub async fn receive_injected_transaction( diff --git a/ethexe/malachite/service/src/starter.rs b/ethexe/malachite/service/src/starter.rs index 017fe154cb4..5630749bc08 100644 --- a/ethexe/malachite/service/src/starter.rs +++ b/ethexe/malachite/service/src/starter.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 use crate::{ - MalachiteService, MalachiteServiceConfig, Mempool, + FastSyncReplayTarget, MalachiteService, MalachiteServiceConfig, Mempool, config::ValidatorConfig, externalities::{EthexeExternalities, ExternalitiesConfig}, types::{ChainHead, MalachiteEvent}, @@ -31,6 +31,7 @@ pub struct MalachiteServiceStarter { validators: HashMap, active_era: u64, core_config: MalachiteCoreConfig, + fast_sync_replay_target: Option, } impl MalachiteServiceStarter { @@ -79,6 +80,7 @@ impl MalachiteServiceStarter { }; let core_config = MalachiteCoreConfig { + env: config.env, listen_addr: config.listen_addr, base: config.home_dir.clone(), persistent_peers: config.persistent_peers.clone(), @@ -107,6 +109,7 @@ impl MalachiteServiceStarter { chain_head: chain_head.clone(), pending_events: Default::default(), event_tx, + fast_sync_replay_filter: RwLock::new(None), }); // On-chain addresses → pub keys, so era rotations resolve back without an out-of-band lookup. @@ -124,9 +127,14 @@ impl MalachiteServiceStarter { validators, active_era, core_config, + fast_sync_replay_target: None, }) } + pub fn enable_fast_sync_replay_filter(&mut self, target: FastSyncReplayTarget) { + self.fast_sync_replay_target = Some(target); + } + /// Launch the consensus core and assemble the running [`MalachiteService`]. pub async fn start(self) -> Result { let MalachiteServiceStarter { @@ -137,13 +145,14 @@ impl MalachiteServiceStarter { validators, active_era, core_config, + fast_sync_replay_target, } = self; let inner = MalachiteCore::new(core_config, externalities.clone()) .await .context("starting ethexe-malachite-core")?; - Ok(MalachiteService { + let service = MalachiteService { events_rx, chain_head, mempool, @@ -151,6 +160,12 @@ impl MalachiteServiceStarter { validators, active_era, inner: Some(inner), - }) + }; + + if let Some(target) = fast_sync_replay_target { + service.enable_fast_sync_replay_filter(target).await?; + } + + Ok(service) } } diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index 2ba2bcb53f1..3af826c8a85 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -238,7 +238,6 @@ async fn single_validator_finalizes_and_recovers_after_restart() { .start() .await .expect("start malachite service"); - svc.start_app_task(); // Feed chain heads one-per-round so the quarantine-advance // probe always sees a strictly newer EB (parent's @@ -286,7 +285,6 @@ async fn single_validator_finalizes_and_recovers_after_restart() { .start() .await .expect("restart malachite service"); - svc2.start_app_task(); let mut pending2 = chain[32..].iter().copied(); let (high2, finalized2) = collect_until_finalized(&mut svc2, &mut pending2, 3, Duration::from_secs(60)).await; diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 34fe9c4992d..819ad97ddbd 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -600,7 +600,7 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { observer, compute, network, - malachite, + malachite_starter, db, #[cfg(test)] sender, @@ -716,15 +716,13 @@ pub(crate) async fn sync(service: &mut Service) -> Result<()> { globals.latest_computed_mb_hash = latest_committed_mb; }); - if let Some(malachite) = malachite.as_mut() { - // `Service::run` performs fast sync before `run_inner().start_app_task()`, - // so live Malachite callbacks cannot race this startup replay gate. - let _ = malachite.enable_fast_sync_replay_filter(replay_target)?; - // Fast sync restores DB globals directly, but Malachite keeps its - // own in-memory chain head for proposal/validation. Seed it now; the - // observer will only deliver later EBs after the app task starts. - malachite.receive_new_chain_head(block_data.to_simple()); - } + // `Service::run` performs fast sync before `run_inner().start_app_task()`, + // so live Malachite callbacks cannot race this startup replay gate. + malachite_starter.enable_fast_sync_replay_filter(replay_target); + // Fast sync restores DB globals directly, but Malachite keeps its + // own in-memory chain head for proposal/validation. Seed it now; the + // observer will only deliver later EBs after the app task starts. + //malachite.receive_new_chain_head(block_data.to_simple()); log::info!( "Fast synchronization done: synced to {latest_committed_eb:?}, height {height:?}, MB {latest_committed_mb}", diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index f76cc954358..bca20e91d0d 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -59,14 +59,8 @@ use ethexe_malachite::{ InjectedTxMempool, MalachiteEvent, MalachiteServiceConfig, MalachiteServiceStarter, ValidatorEntry, }; -use ethexe_network::{ - NetworkEvent, NetworkRuntimeConfig, NetworkService, TransportType, - db_sync::ExternalDataProvider, -}; -use ethexe_observer::{ - ObserverConfig, ObserverEvent, ObserverService, - utils::{BlockId, BlockLoader}, -}; +use ethexe_network::{NetworkEvent, NetworkRuntimeConfig, NetworkService, TransportType}; +use ethexe_observer::{ObserverConfig, ObserverEvent, ObserverService, utils::BlockLoader}; use ethexe_processor::{ProcessedCodeInfo, Processor, ProcessorConfig, ValidCodeInfo}; use ethexe_prometheus::{PrometheusEvent, PrometheusService}; use ethexe_rpc::{RpcEvent, RpcServer}; diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index 7eda01d4e83..846f3f33c3c 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -39,9 +39,9 @@ use ethexe_ethereum::{ router::RouterQuery, }; use ethexe_malachite::{ - InjectedTxMempool, MalachiteServiceConfig, MalachiteServiceStarter, - Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, derive_libp2p_secret, - malachite_libp2p_peer_id, + FastSyncReplayTarget, InjectedTxMempool, MalachiteConfigEnvironment, MalachiteServiceConfig, + MalachiteServiceStarter, Multiaddr as MalachiteMultiaddr, PeerId, ValidatorEntry, + derive_libp2p_secret, malachite_libp2p_peer_id, }; use ethexe_network::{NetworkConfig, NetworkRuntimeConfig, NetworkService, export::Multiaddr}; use ethexe_observer::{