diff --git a/.config/nextest.toml b/.config/nextest.toml index 5b47a1d07c0..329edbd6f48 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -27,6 +27,9 @@ test-group = 'authorship' [[profile.default.overrides]] filter = 'package(ethexe-service)' leak-timeout = { period = "10s", result = "pass" } +# Heavy multi-validator tests (4 Anvil + 4 malachite engines) starve under +# full-suite parallelism and blow the 120s cap; reserve threads like CI does. +threads-required = 4 [profile.default.junit] path = "junit.xml" diff --git a/Cargo.lock b/Cargo.lock index 99216428672..814f639b42d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1224,7 +1224,7 @@ version = "0.7.0-pre" source = "git+https://github.com/circlefin/malachite?rev=1fe7961aca933cefad8e4d9a52f50eda565288e7#1fe7961aca933cefad8e4d9a52f50eda565288e7" dependencies = [ "bs58 0.5.1", - "multihash 0.19.3", + "multihash 0.19.5", "rand 0.8.5", "serde", "thiserror 2.0.17", @@ -2412,6 +2412,32 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +[[package]] +name = "beetswap" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a920844a4696d53e72190a8348f0048a6799647b3448d38bef90b85e0310df0" +dependencies = [ + "asynchronous-codec 0.7.0", + "blockstore", + "bytes", + "cid 0.11.3", + "fnv", + "futures-core", + "futures-timer", + "futures-util", + "libp2p-core 0.43.2", + "libp2p-identity", + "libp2p-swarm 0.47.0", + "multihash-codetable", + "quick-protobuf", + "smallvec", + "thiserror 2.0.17", + "tracing", + "unsigned-varint 0.8.0", + "web-time", +] + [[package]] name = "bimap" version = "0.6.3" @@ -2637,6 +2663,18 @@ dependencies = [ "piper", ] +[[package]] +name = "blockstore" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509096e88e431095763b3f5ee1e1cdb09212e4d5b2eccc91ddea965deefedb7d" +dependencies = [ + "cid 0.11.3", + "dashmap 6.1.0", + "multihash 0.19.5", + "thiserror 2.0.17", +] + [[package]] name = "blst" version = "0.3.16" @@ -3186,6 +3224,17 @@ dependencies = [ "unsigned-varint 0.7.2", ] +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash 0.19.5", + "unsigned-varint 0.8.0", +] + [[package]] name = "cipher" version = "0.2.5" @@ -5875,19 +5924,21 @@ dependencies = [ "assert_matches", "async-trait", "auto_impl", + "beetswap", + "blockstore", "bytes", + "cid 0.11.3", "derive_more 2.1.1", + "either", "ethexe-common", "ethexe-db", "ethexe-malachite-core", - "ethexe-service-utils", "futures", "gear-workspace-hack", "gprimitives", "gsigner", "indexmap 2.14.0", "ip_network", - "itertools 0.13.0", "libp2p 0.56.0", "libp2p-gossipsub", "libp2p-scatter", @@ -5896,6 +5947,7 @@ dependencies = [ "lru 0.16.3", "metrics", "metrics-derive", + "multihash 0.19.5", "nonempty 0.12.0", "parity-scale-codec", "prometheus-client 0.23.1", @@ -5954,7 +6006,6 @@ version = "2.0.0" dependencies = [ "alloy", "anyhow", - "derive_more 2.1.1", "ethexe-common", "ethexe-db", "ethexe-ethereum", @@ -6128,6 +6179,7 @@ dependencies = [ "demo-async-init", "demo-delayed-sender-ethexe", "demo-fungible-token", + "demo-mul-by-const", "demo-piggy-bank", "demo-ping", "demo-reply-callback", @@ -7949,6 +8001,7 @@ dependencies = [ name = "gear-workspace-hack" version = "0.1.0" dependencies = [ + "aead", "ahash", "alloy", "alloy-chains", @@ -7993,10 +8046,12 @@ dependencies = [ "byteorder", "bytes", "cc", + "cipher 0.4.4", "clap 4.5.54", "clap_builder", "concurrent-queue", "const-hex", + "core2", "cranelift-bitset", "cranelift-codegen", "cranelift-codegen-meta", @@ -8099,7 +8154,7 @@ dependencies = [ "merlin", "miniz_oxide", "mio", - "multihash 0.19.3", + "multihash 0.19.5", "names", "nix", "nom 7.1.3", @@ -10373,7 +10428,7 @@ dependencies = [ "libp2p-identity", "log", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "multistream-select", "once_cell", "parking_lot 0.12.5", @@ -10399,7 +10454,7 @@ dependencies = [ "futures-timer", "libp2p-identity", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "multistream-select", "parking_lot 0.12.5", "pin-project", @@ -10531,7 +10586,7 @@ dependencies = [ "ed25519-dalek", "hkdf", "k256", - "multihash 0.19.3", + "multihash 0.19.5", "p256", "quick-protobuf", "rand 0.8.5", @@ -10689,7 +10744,7 @@ dependencies = [ "libp2p-identity", "log", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "once_cell", "quick-protobuf", "rand 0.8.5", @@ -10713,7 +10768,7 @@ dependencies = [ "libp2p-core 0.43.2", "libp2p-identity", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "quick-protobuf", "rand 0.8.5", "snow", @@ -11939,7 +11994,7 @@ dependencies = [ "data-encoding", "libp2p-identity", "multibase", - "multihash 0.19.3", + "multihash 0.19.5", "percent-encoding", "serde", "static_assertions", @@ -11970,7 +12025,7 @@ dependencies = [ "blake3", "core2", "digest 0.10.7", - "multihash-derive", + "multihash-derive 0.8.1", "sha2 0.10.9", "sha3", "unsigned-varint 0.7.2", @@ -11987,7 +12042,7 @@ dependencies = [ "blake3", "core2", "digest 0.10.7", - "multihash-derive", + "multihash-derive 0.8.1", "sha2 0.10.9", "sha3", "unsigned-varint 0.7.2", @@ -11995,15 +12050,33 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.3" +version = "0.19.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" dependencies = [ - "core2", "serde", "unsigned-varint 0.8.0", ] +[[package]] +name = "multihash-codetable" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67996849749d25f1da9f238e8ace2ece8f9d6bdf3f9750aaf2ae7de3a5cad8ea" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "blake3", + "core2", + "digest 0.10.7", + "multihash-derive 0.9.3", + "ripemd", + "sha1", + "sha2 0.10.9", + "sha3", + "strobe-rs", +] + [[package]] name = "multihash-derive" version = "0.8.1" @@ -12018,6 +12091,29 @@ dependencies = [ "synstructure 0.12.6", ] +[[package]] +name = "multihash-derive" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4723acdce756db211a53f01b3419dbdf63cb48cef5df86260f55309364735fbf" +dependencies = [ + "multihash 0.19.5", + "multihash-derive-impl", +] + +[[package]] +name = "multihash-derive-impl" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f932556f78452e5604cef711349d337ec081a9aa3c96e67b3127c8f5df05d550" +dependencies = [ + "proc-macro-crate 3.4.0", + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure 0.13.2", +] + [[package]] name = "multimap" version = "0.8.3" @@ -15930,7 +16026,7 @@ dependencies = [ "libp2p 0.52.4", "linked_hash_set", "log", - "multihash 0.19.3", + "multihash 0.19.5", "parity-scale-codec", "prost 0.12.6", "prost-build 0.12.6", @@ -16576,7 +16672,7 @@ dependencies = [ "litep2p", "log", "multiaddr 0.18.2", - "multihash 0.19.3", + "multihash 0.19.5", "rand 0.8.5", "thiserror 1.0.69", "zeroize", @@ -18808,6 +18904,19 @@ dependencies = [ "serde", ] +[[package]] +name = "strobe-rs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98fe17535ea31344936cc58d29fec9b500b0452ddc4cc24c429c8a921a0e84e5" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "keccak", + "subtle 2.6.1", + "zeroize", +] + [[package]] name = "strsim" version = "0.8.0" diff --git a/ethexe/compute/src/compute.rs b/ethexe/compute/src/compute.rs index e7eb682843a..73a6d880667 100644 --- a/ethexe/compute/src/compute.rs +++ b/ethexe/compute/src/compute.rs @@ -13,8 +13,8 @@ use crate::{ComputeError, ComputeEvent, ProcessorExt, Result, service::SubServic use ethexe_common::{ PromiseEmissionMode, PromisePolicy, SimpleBlockData, db::{ - CodesStorageRW, CompactMb, ConfigStorageRO, GlobalsStorageRO, MbStorageRO, MbStorageRW, - OnChainStorageRO, + BlockMetaStorageRO, CodesStorageRW, CompactMb, ConfigStorageRO, GlobalsStorageRO, + MbStorageRO, MbStorageRW, OnChainStorageRO, }, events::BlockRequestEvent, injected::Promise, @@ -85,6 +85,13 @@ pub struct ComputeSubService { metrics: Metrics, input: VecDeque, + /// Requests whose prerequisite EB (the block this MB advances to) is not + /// yet prepared in the DB. Held here instead of executing — and moved back + /// into `input` by [`Self::receive_prepared_block`] once the prerequisite + /// lands. This is the gate Malachite used to apply before emitting events; + /// owning it here lets replayed/early MBs flow through as events while + /// their execution waits for the code-validation pipeline to catch up. + deferred: VecDeque, /// Head of the in-flight computation, kept so [`Self::receive_mb`] can /// skip duplicates that would otherwise re-emit `MbComputed`. in_flight_mb: Option, @@ -111,6 +118,7 @@ impl ComputeSubService

{ promise_emission_mode, metrics: Metrics::default(), input: VecDeque::new(), + deferred: VecDeque::new(), in_flight_mb: None, computation: None, promises_stream: None, @@ -119,12 +127,13 @@ impl ComputeSubService

{ } pub fn receive_mb(&mut self, mb_hash: H256, promise_policy: PromisePolicy) { - // Idempotent: skip if already computed, in flight, or queued — - // otherwise BlockProposal+BlockFinalized for the same head emit - // `MbComputed` twice. + // Idempotent: skip if already computed, in flight, or queued + // (`input` or `deferred`) — otherwise BlockProposal+BlockFinalized + // for the same head emit `MbComputed` twice. if self.db.mb_meta(mb_hash).computed || self.in_flight_mb == Some(mb_hash) || self.input.iter().any(|r| r.mb_hash == mb_hash) + || self.deferred.iter().any(|r| r.mb_hash == mb_hash) { return; } @@ -134,6 +143,32 @@ impl ComputeSubService

{ }); } + /// An Ethereum block has been prepared: requeue every deferred request + /// whose prerequisite EB is now satisfied. `PrepareSubService` prepares a + /// whole ancestor chain but only reports the head, so we re-check all + /// deferred requests rather than matching the reported hash. + pub fn receive_prepared_block(&mut self, _eb_hash: H256) { + let mut still_deferred = VecDeque::with_capacity(self.deferred.len()); + while let Some(req) = self.deferred.pop_front() { + if self.db.mb_meta(req.mb_hash).computed { + continue; + } + if self.prerequisite_ready(req.mb_hash) { + self.input.push_back(req); + } else { + still_deferred.push_back(req); + } + } + self.deferred = still_deferred; + } + + /// Whether the EB this MB advances to is prepared (or there is none). + /// Execution reads that EB's events, so it must wait until then. + fn prerequisite_ready(&self, mb_hash: H256) -> bool { + let eb = self.db.mb_meta(mb_hash).last_advanced_eb; + eb.is_zero() || self.db.block_meta(eb).prepared + } + async fn compute( db: Database, mut processor: P, @@ -443,25 +478,37 @@ impl SubService for ComputeSubService

{ type Output = ComputeEvent; fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll> { - // (1) Pick up the next request whenever no work is in flight. + // (1) Pick up the next ready request whenever no work is in flight. + // Skip already-computed requests and park ones whose prerequisite EB + // is not prepared yet into `deferred`, scanning past them so a parked + // request never head-of-line blocks a ready one behind it. if self.computation.is_none() && self.promises_stream.is_none() && self.pending_event.is_none() - && let Some(req) = self.input.pop_front() { - let (sender, receiver) = mpsc::unbounded_channel(); - self.in_flight_mb = Some(req.mb_hash); - self.promises_stream = Some(MbPromisesStream { receiver }); - self.computation = Some(future_timing::timed( - Self::compute( - self.db.clone(), - self.processor.clone(), - req, - self.promise_emission_mode, - sender, - ) - .boxed(), - )); + while let Some(req) = self.input.pop_front() { + if self.db.mb_meta(req.mb_hash).computed { + continue; + } + if !self.prerequisite_ready(req.mb_hash) { + self.deferred.push_back(req); + continue; + } + let (sender, receiver) = mpsc::unbounded_channel(); + self.in_flight_mb = Some(req.mb_hash); + self.promises_stream = Some(MbPromisesStream { receiver }); + self.computation = Some(future_timing::timed( + Self::compute( + self.db.clone(), + self.processor.clone(), + req, + self.promise_emission_mode, + sender, + ) + .boxed(), + )); + break; + } } // (2) Forward streaming promises before anything else so the @@ -557,6 +604,11 @@ mod tests { }, ); db.set_block_events(hash, &events); + // Compute now defers an MB until the EB it advances to is prepared, so + // these synthetic EBs (standing in for already-synced blocks) must carry + // the flag for the direct `ComputeSubService` tests that never pump a + // `BlockPrepared` notification. + db.mutate_block_meta(hash, |m| m.prepared = true); hash } diff --git a/ethexe/compute/src/lib.rs b/ethexe/compute/src/lib.rs index a113d394c69..06dfdaf476b 100644 --- a/ethexe/compute/src/lib.rs +++ b/ethexe/compute/src/lib.rs @@ -189,6 +189,9 @@ pub enum ComputeError { #[error(transparent)] Processor(#[from] ProcessorError), + + #[error("other: {0}")] + Other(&'static str), } type Result = std::result::Result; diff --git a/ethexe/compute/src/service.rs b/ethexe/compute/src/service.rs index e79cbfac70d..6208db06921 100644 --- a/ethexe/compute/src/service.rs +++ b/ethexe/compute/src/service.rs @@ -98,6 +98,11 @@ impl Stream for ComputeService

{ }; if let Poll::Ready(result) = self.prepare_sub_service.poll_next(cx) { + // A freshly prepared EB may unblock MBs that were deferred + // waiting for it (the gate Malachite used to own). + if let Ok(crate::prepare::Event::BlockPrepared(hash)) = &result { + self.mb_compute_sub_service.receive_prepared_block(*hash); + } return Poll::Ready(Some(result.map(ComputeEvent::from))); }; @@ -135,8 +140,8 @@ mod tests { use ethexe_common::{ BlockHeader, CodeAndIdUnchecked, db::{ - BlockMetaStorageRO, CodesStorageRO, CompactMb, MbStorageRO, MbStorageRW, - OnChainStorageRW, + BlockMetaStorageRO, BlockMetaStorageRW, CodesStorageRO, CompactMb, MbStorageRO, + MbStorageRW, OnChainStorageRW, }, malachite::{Operation, Operations}, mock::{Tap, seed_genesis_zero_mb}, @@ -172,6 +177,9 @@ mod tests { }, ); db.set_block_events(eth_block_hash, &[]); + // Compute defers an MB until its advanced EB is prepared; mark it so the + // `compute_mb` test (which never pumps a `BlockPrepared`) runs immediately. + db.mutate_block_meta(eth_block_hash, |m| m.prepared = true); let operations_hash = db.set_operations(Operations::new(vec![ Operation::AdvanceTillEthereumBlock { diff --git a/ethexe/ethereum/src/lib.rs b/ethexe/ethereum/src/lib.rs index 3cf5b418782..e2d59164b8b 100644 --- a/ethexe/ethereum/src/lib.rs +++ b/ethexe/ethereum/src/lib.rs @@ -72,7 +72,7 @@ use crate::{ }; use alloy::{ consensus::SignableTransaction, - eips::BlockId, + eips::{BlockId, BlockNumberOrTag}, network::{self, Ethereum as AlloyEthereum, EthereumWallet, Network, TxSigner}, primitives::{Address as AlloyAddress, B256, ChainId, Signature, U256 as AlloyU256, address}, providers::{ @@ -714,6 +714,29 @@ pub(crate) use signatures_consts; use crate::wvara::WVara; +/// A helping trait for converting various types into `alloy::eips::BlockNumberOrTag`. +pub trait IntoBlockNumberOrTag { + fn into_block_number_or_tag(self) -> BlockNumberOrTag; +} + +impl IntoBlockNumberOrTag for u32 { + fn into_block_number_or_tag(self) -> BlockNumberOrTag { + BlockNumberOrTag::Number(self as u64) + } +} + +impl IntoBlockNumberOrTag for u64 { + fn into_block_number_or_tag(self) -> BlockNumberOrTag { + BlockNumberOrTag::Number(self) + } +} + +impl IntoBlockNumberOrTag for BlockNumberOrTag { + fn into_block_number_or_tag(self) -> BlockNumberOrTag { + self + } +} + /// A helping trait for converting various types into `alloy::eips::BlockId`. pub trait IntoBlockId { fn into_block_id(self) -> BlockId; diff --git a/ethexe/ethereum/src/router/events.rs b/ethexe/ethereum/src/router/events.rs index 9d092cddf4f..f226898326a 100644 --- a/ethexe/ethereum/src/router/events.rs +++ b/ethexe/ethereum/src/router/events.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 use crate::{ - IRouter, + IRouter, IntoBlockNumberOrTag, abi::utils::{bytes32_to_code_id, bytes32_to_h256}, decode_log, router::RouterQuery, @@ -218,31 +218,47 @@ impl<'a> EBCommittedEventBuilder<'a> { pub struct CodeGotValidatedEventBuilder<'a> { event: Event<&'a RootProvider, IRouter::CodeGotValidated>, - valid: Option, } impl<'a> CodeGotValidatedEventBuilder<'a> { pub(crate) fn new(query: &'a RouterQuery) -> Self { Self { event: query.instance.CodeGotValidated_filter(), - valid: None, } } pub fn valid(mut self, valid: bool) -> Self { - self.valid = Some(valid); + self.event = self.event.topic1(valid); self } + pub fn from_block(mut self, block: impl IntoBlockNumberOrTag) -> Self { + self.event = self.event.from_block(block.into_block_number_or_tag()); + self + } + + pub fn to_block(mut self, block: impl IntoBlockNumberOrTag) -> Self { + self.event = self.event.to_block(block.into_block_number_or_tag()); + self + } + + pub async fn query(self) -> Result> { + Ok(self + .event + .chunked() + .query() + .await? + .into_iter() + .map(|(event, log)| (event.into(), log)) + .collect()) + } + pub async fn subscribe( self, ) -> Result> + Unpin + use<>> { - let mut event = self.event; - if let Some(valid) = self.valid { - event = event.topic1(valid); - } - Ok(event + Ok(self + .event .subscribe() .await? .into_stream() @@ -337,31 +353,47 @@ impl<'a> ComputationSettingsChangedEventBuilder<'a> { pub struct ProgramCreatedEventBuilder<'a> { event: Event<&'a RootProvider, IRouter::ProgramCreated>, - code_id: Option, } impl<'a> ProgramCreatedEventBuilder<'a> { pub(crate) fn new(query: &'a RouterQuery) -> Self { Self { event: query.instance.ProgramCreated_filter(), - code_id: None, } } pub fn code_id(mut self, code_id: CodeId) -> Self { - self.code_id = Some(code_id); + let code_id: B256 = code_id.into_bytes().into(); + self.event = self.event.topic1(code_id); self } + pub fn from_block(mut self, block: impl IntoBlockNumberOrTag) -> Self { + self.event = self.event.from_block(block.into_block_number_or_tag()); + self + } + + pub fn to_block(mut self, block: impl IntoBlockNumberOrTag) -> Self { + self.event = self.event.to_block(block.into_block_number_or_tag()); + self + } + + pub async fn query(self) -> Result> { + Ok(self + .event + .chunked() + .query() + .await? + .into_iter() + .map(|(event, log)| (event.into(), log)) + .collect()) + } + pub async fn subscribe( self, ) -> Result> + Unpin + use<>> { - let mut event = self.event; - if let Some(code_id) = self.code_id { - let code_id: B256 = code_id.into_bytes().into(); - event = event.topic1(code_id); - } - Ok(event + Ok(self + .event .subscribe() .await? .into_stream() diff --git a/ethexe/malachite/core/src/app.rs b/ethexe/malachite/core/src/app.rs index 4e7d06bc656..dcd664d749f 100644 --- a/ethexe/malachite/core/src/app.rs +++ b/ethexe/malachite/core/src/app.rs @@ -422,7 +422,6 @@ where // proposer side. cascade_save guarantees ancestor-first // ordering against the application. self.record_assembled_block(block).await?; - Ok(locally) } @@ -778,7 +777,7 @@ mod tests { ConsensusRequest, NetworkRequest, app::{events::TxEvent, streaming::StreamId}, }; - use std::time::Duration; + use std::{sync::Arc, time::Duration}; use tempfile::TempDir; use tokio::sync::mpsc; @@ -897,8 +896,8 @@ mod tests { /// `process_received_proposal_part`, returning the final reply /// value (`Some(proposed)` on the same-height happy path, `None` /// when the parts were dropped or buffered). - async fn run_stream( - handler: &mut AppMsgHandler, + async fn run_stream( + handler: &mut AppMsgHandler, peer: PeerId, stream: Vec>, ) -> Option> { diff --git a/ethexe/malachite/core/src/config.rs b/ethexe/malachite/core/src/config.rs index ffb863ec941..9ea1c5d493b 100644 --- a/ethexe/malachite/core/src/config.rs +++ b/ethexe/malachite/core/src/config.rs @@ -34,10 +34,18 @@ pub enum NodeRole { FullNode, } +#[derive(Debug, Copy, Clone, Default)] +pub enum Environment { + #[default] + Production, + Test, +} + /// All configuration the service needs to bootstrap the malachite engine. /// Application-specific knobs live behind [`crate::Externalities`]. #[derive(Clone, Debug)] pub struct MalachiteCoreConfig { + pub env: Environment, /// Base directory; the service owns `/malachite/` (consensus WAL /// + RocksDB store), created on first run and resumed on restarts. pub base: PathBuf, diff --git a/ethexe/malachite/core/src/lib.rs b/ethexe/malachite/core/src/lib.rs index 936fed1e5e7..927e0af837d 100644 --- a/ethexe/malachite/core/src/lib.rs +++ b/ethexe/malachite/core/src/lib.rs @@ -89,7 +89,10 @@ mod streaming; pub use crate::{ codec::ScaleCodec, - config::{MalachiteCoreConfig, Multiaddr, NodeRole, ValidatorEntry}, + config::{ + Environment as MalachiteConfigEnvironment, MalachiteCoreConfig, Multiaddr, NodeRole, + ValidatorEntry, + }, context::MalachiteCtx, externalities::Externalities, service::{MService, MalachiteCore, MalachiteNetworkParts}, diff --git a/ethexe/malachite/core/src/service.rs b/ethexe/malachite/core/src/service.rs index 4772d60611e..9fd3c771a40 100644 --- a/ethexe/malachite/core/src/service.rs +++ b/ethexe/malachite/core/src/service.rs @@ -6,13 +6,13 @@ use crate::{ app, codec::ScaleCodec, - config::{MalachiteCoreConfig, NodeRole}, + config::{Environment, MalachiteCoreConfig, NodeRole}, context::{MalachiteCtx, Validator, ValidatorSet}, externalities::Externalities, signing::{MalachiteSigner, private_key_from_gsigner, public_key_from_gsigner}, state::{SharedValidatorSet, State}, store::Store, - types::Address, + types::{Address, H256}, }; use advisory_lock::{AdvisoryFileLock, FileLockMode}; use anyhow::{Context as _, Result}; @@ -66,6 +66,10 @@ pub struct MalachiteCore { wal_path: PathBuf, /// Shared with the app loop; [`Self::update_validators`] writes here. validator_set: SharedValidatorSet, + /// Persistent app store, shared with the app task. Kept here for + /// startup decisions that need to inspect replay state before the + /// app task is released. + store: Store, /// Validator proof bytes intended for network. validator_proof: Option, /// Keeps the externalities alive for the app task. @@ -213,7 +217,7 @@ impl MalachiteCore { }; // ---- engine ---- - let inner_cfg = build_inner_config(&moniker); + let inner_cfg = build_inner_config(&moniker, config.env); let ctx = MalachiteCtx::new(); let consensus_signer = MalachiteSigner::new(signer.private_key().clone()); let (channels, engine) = EngineBuilder::new(ctx.clone(), inner_cfg) @@ -236,7 +240,7 @@ impl MalachiteCore { signer, validator_set.clone(), address, - store, + store.clone(), config.propose_timeout, )?; @@ -256,11 +260,17 @@ impl MalachiteCore { app_handle, wal_path, validator_set, + store, validator_proof, _externalities: externalities, }) } + /// Whether `block_hash` is already finalized in the persistent app store. + pub fn is_finalized(&self, block_hash: H256) -> Result { + self.store.is_finalized(block_hash) + } + /// Validator proof bytes intended for network. pub fn validator_proof(&self) -> Option { self.validator_proof.clone() @@ -304,7 +314,7 @@ impl FusedStream for MalachiteCore { impl MService for MalachiteCore {} -fn build_inner_config(moniker: &str) -> InnerNodeConfig { +fn build_inner_config(moniker: &str, env: Environment) -> InnerNodeConfig { let consensus = ConsensusConfig { enabled: true, value_payload: ValuePayload::ProposalAndParts, @@ -312,10 +322,23 @@ fn build_inner_config(moniker: &str) -> InnerNodeConfig { // NOTE: the config is actually unused because we have our own network implementation p2p: P2pConfig::default(), }; + let value_sync = match env { + Environment::Production => ValueSyncConfig::default(), + Environment::Test => ValueSyncConfig { + // Service tests do not run Malachite's application task forever, so + // use short value-sync waits and small batches to make missing replay + // data fail fast instead of blocking the event queue. + status_update_interval: Duration::from_millis(500), + request_timeout: Duration::from_secs(3), + parallel_requests: 16, + batch_size: 32, + ..Default::default() + }, + }; InnerNodeConfig { moniker: moniker.to_string(), consensus, - value_sync: ValueSyncConfig::default(), + value_sync, logging: LoggingConfig::default(), metrics: MetricsConfig::default(), runtime: RuntimeConfig::default(), diff --git a/ethexe/malachite/core/src/store.rs b/ethexe/malachite/core/src/store.rs index 109b2b2ba95..33a528911ed 100644 --- a/ethexe/malachite/core/src/store.rs +++ b/ethexe/malachite/core/src/store.rs @@ -35,8 +35,9 @@ use parity_scale_codec::{Decode, Encode}; use rocksdb::{DB, Options, WriteBatch}; use crate::{ + BlockPayload, context::Height, - types::{Block, BlockPayload, CommitCertificate, H256}, + types::{Block, CommitCertificate, H256}, }; mod prefix { @@ -325,6 +326,11 @@ impl Store { } } + /// Whether `block_hash` is known and marked finalized. + pub fn is_finalized(&self, block_hash: H256) -> Result { + Ok(self.get_block(block_hash)?.is_some_and(|e| e.finalized)) + } + /// Drive `save_fn` over every ancestor that is now ready, starting from /// each seed, then cascade to children so descendants waiting on a gap /// get flushed once it closes. @@ -909,6 +915,20 @@ mod tests { assert!(err.to_string().contains("not saved yet")); } + #[test] + fn is_finalized_tracks_persistent_finalization_state() { + let (_d, store) = open_store(); + + assert!(!store.is_finalized(h(1)).unwrap()); + + store.insert_block(mk_entry(h(1), H256::zero(), 1)).unwrap(); + store.mark_saved(h(1)).unwrap(); + assert!(!store.is_finalized(h(1)).unwrap()); + + store.mark_finalized(h(1), mk_cert(1, h(1))).unwrap(); + assert!(store.is_finalized(h(1)).unwrap()); + } + // --- restart persistence -------------------------------------------- #[test] diff --git a/ethexe/malachite/core/tests/multi_validators.rs b/ethexe/malachite/core/tests/multi_validators.rs index a2e05c841de..16fedee1c90 100644 --- a/ethexe/malachite/core/tests/multi_validators.rs +++ b/ethexe/malachite/core/tests/multi_validators.rs @@ -292,6 +292,7 @@ fn build_config_with_role( role: NodeRole, ) -> MalachiteCoreConfig { MalachiteCoreConfig { + env: Default::default(), base: setup.home.path().to_path_buf(), validator_secret: setup.private_key.clone(), validators, diff --git a/ethexe/malachite/service/src/config.rs b/ethexe/malachite/service/src/config.rs index 4c32cad5c97..88700a0bf64 100644 --- a/ethexe/malachite/service/src/config.rs +++ b/ethexe/malachite/service/src/config.rs @@ -3,19 +3,22 @@ //! Top-level configuration of the [`crate::MalachiteService`]. //! -//! User-facing knobs (persistent peers, gas allowance, quarantine -//! depth, **validator set**) live here. The validator set +//! User-facing knobs (gas allowance, quarantine depth, **validator set**) +//! live here. The validator set //! is wired in directly — there is no separate genesis file — so the //! caller is the single source of truth for who can vote. pub use ethexe_malachite_core::ValidatorEntry; use crate::Mempool; +use ethexe_malachite_core::MalachiteConfigEnvironment; use gsigner::{PublicKey, secp256k1::Signer}; use std::{path::PathBuf, time::Duration}; #[derive(Clone, Debug)] pub struct MalachiteServiceConfig { + pub env: MalachiteConfigEnvironment, + /// Gas allowance per block. pub gas_allowance: u64, @@ -58,6 +61,7 @@ impl MalachiteServiceConfig { /// The validator set is left empty — fill it in with [`Self::with_validators`]. pub fn from_home_dir(home_dir: PathBuf) -> Self { Self { + env: Default::default(), gas_allowance: Self::DEFAULT_GAS_ALLOWANCE, canonical_quarantine: Self::DEFAULT_CANONICAL_QUARANTINE, post_quarantine_delay: Self::DEFAULT_POST_QUARANTINE_DELAY, diff --git a/ethexe/malachite/service/src/externalities.rs b/ethexe/malachite/service/src/externalities.rs index a1e0894b96c..b18ded85fcb 100644 --- a/ethexe/malachite/service/src/externalities.rs +++ b/ethexe/malachite/service/src/externalities.rs @@ -57,8 +57,10 @@ use ethexe_db::Database; use ethexe_malachite_core::{Block, BlockPayload, Externalities, MAX_BLOCK_PAYLOAD_BYTES}; use gprimitives::H256; use parity_scale_codec::{DecodeAll, Encode}; -use std::{collections::VecDeque, sync::Arc}; -use tokio::sync::{RwLock, mpsc::UnboundedSender}; +use std::sync::Arc; +#[cfg(test)] +use tokio::sync::RwLock; +use tokio::sync::mpsc::UnboundedSender; use tracing::{debug, error, trace, warn}; /// Constant parameters for [`EthexeExternalities`]; @@ -81,23 +83,10 @@ pub(crate) struct EthexeExternalities { pub mempool: Option>, /// Reference to the latest chain head data. pub chain_head: Arc, - /// Pending service events queue. - /// Release events from here only when their prerequisite EB is prepared. - pub pending_events: RwLock>, /// Channel to poll events in MalachiteService. pub event_tx: UnboundedSender>, } -/// One outbound [`MalachiteEvent`] that can't be released until its -/// `prerequisite` Eth block is prepared in local DB. -pub(crate) struct PendingEvent { - /// Event body - pub event: MalachiteEvent, - /// Prerequisite Eth block hash - /// that must be prepared before this event can be emitted - pub prerequisite: H256, -} - #[async_trait] impl Externalities for EthexeExternalities { async fn process_mb_proposal(&self, mb_hash: H256, mb: Block) -> Result<()> { @@ -132,14 +121,15 @@ impl Externalities for EthexeExternalities { meta.last_advanced_eb = last_advanced; }); - self.try_emit_or_queue( - MalachiteEvent::BlockProposal { - height: mb.height, - mb_hash, - }, - last_advanced, - ) - .await; + // Emit immediately: the "wait until the advanced EB is prepared" + // gating now lives in ethexe-compute, which defers MB execution + // (not event emission) until the prerequisite EB is ready. Emitting + // here unconditionally keeps events strictly ordered and lets the + // service drain fast-sync replay without head-of-line stalls. + let _ = self.event_tx.send(Ok(MalachiteEvent::BlockProposal { + height: mb.height, + mb_hash, + })); Ok(()) } @@ -173,7 +163,6 @@ impl Externalities for EthexeExternalities { pool.forget(&injected).await; } } - self.db .globals_mutate(|g| g.latest_finalized_mb_hash = mb_hash); @@ -182,16 +171,12 @@ impl Externalities for EthexeExternalities { mb_hash, signatures: cert.signatures, }; - let last_advanced = self.db.mb_meta(mb_hash).last_advanced_eb; - self.try_emit_or_queue( - MalachiteEvent::BlockFinalized { - cert: app_cert, - height: cert.height, - mb_hash, - }, - last_advanced, - ) - .await; + // Emit immediately; see the note in `process_mb_proposal`. + let _ = self.event_tx.send(Ok(MalachiteEvent::BlockFinalized { + cert: app_cert, + height: cert.height, + mb_hash, + })); Ok(()) } @@ -470,39 +455,6 @@ impl Externalities for EthexeExternalities { } impl EthexeExternalities { - /// Check whether the prerequisite EB is prepared in local DB. - /// Zero hash is a special case that always passes. - fn prerequisite_satisfied(&self, prerequisite: H256) -> bool { - use ethexe_common::db::BlockMetaStorageRO; - prerequisite.is_zero() || self.db.block_meta(prerequisite).prepared - } - - /// Send event immediately if prerequisite is satisfied, otherwise queue it for later emission. - pub(crate) async fn try_emit_or_queue(&self, event: MalachiteEvent, prerequisite: H256) { - let mut queue = self.pending_events.write().await; - if queue.is_empty() && self.prerequisite_satisfied(prerequisite) { - // Channel receiver dropped only on shutdown — best-effort. - let _ = self.event_tx.send(Ok(event)); - } else { - queue.push_back(PendingEvent { - event, - prerequisite, - }); - } - } - - /// Check the pending events queue and release any events whose prerequisites are now satisfied. - pub(crate) async fn drain_pending_events(&self) { - let mut queue = self.pending_events.write().await; - while let Some(front) = queue.front() { - if !self.prerequisite_satisfied(front.prerequisite) { - break; - } - let entry = queue.pop_front().expect("just peeked"); - let _ = self.event_tx.send(Ok(entry.event)); - } - } - // Wait for either a new EB candidate past quarantine // or any suitable injected tx to include in the next proposal. async fn wait_for_proposable_content( @@ -640,7 +592,6 @@ mod tests { mempool: Some(Arc::new(EmptyMempool)), chain_head: make_chain_head(), event_tx, - pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 0, @@ -1055,7 +1006,6 @@ mod tests { mempool: Some(Arc::clone(&tracker) as Arc), chain_head: make_chain_head(), event_tx, - pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 0, @@ -1122,7 +1072,6 @@ mod tests { mempool: Some(mempool as Arc), chain_head: make_chain_head(), event_tx, - pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 0, @@ -1876,7 +1825,6 @@ mod tests { mempool: Some(Arc::new(EmptyMempool)), chain_head: make_chain_head(), event_tx, - pending_events: RwLock::new(VecDeque::new()), cfg: ExternalitiesConfig { gas_allowance: 1_000_000, canonical_quarantine: 2, diff --git a/ethexe/malachite/service/src/lib.rs b/ethexe/malachite/service/src/lib.rs index 88dcf6e8418..4b283ca60c0 100644 --- a/ethexe/malachite/service/src/lib.rs +++ b/ethexe/malachite/service/src/lib.rs @@ -28,7 +28,7 @@ //! - [`TxValidity`] (enum) — Validity verdict: `Valid`, `Duplicate`, `Outdated`, … //! //! Driver methods on [`MalachiteService`]: `receive_injected_transaction`, -//! `receive_new_eb`, `receive_eb_synced`, `receive_eb_prepared`, `shutdown`. +//! `receive_new_eb`, `receive_eb_synced`, `shutdown`. //! //! [`TxValidity`] gates inclusion: a producer drops any non-`Valid` tx when //! building an MB, and a validator rejects an entire MB that contains one. @@ -44,9 +44,8 @@ //! height; both series are emitted ancestor-first. //! - Tendermint's quorum threshold is `> 2/3` of total voting power across the //! validator list. -//! - Peer discovery is disabled: every `persistent_peers` multiaddr must include a -//! `/p2p/` suffix, and every validator must be listed or transitively -//! reachable through a listed peer. +//! - Network I/O is supplied by `ethexe-network`; every validator must be listed +//! in the configured validator set. //! - `Drop` is best-effort; call `shutdown().await` before an immediate restart so //! RocksDB locks and sockets release. @@ -61,10 +60,10 @@ mod types; pub use crate::{ config::{MalachiteServiceConfig, ValidatorConfig, ValidatorEntry}, - mempool::{InjectedTxMempool, Mempool, TxInsertionStatus}, + mempool::{DEFAULT_POOL_CAPACITY, InjectedTxMempool, Mempool, TxInsertionStatus}, service::MalachiteService, starter::MalachiteServiceStarter, - tx_validity::{TxValidity, TxValidityChecker}, + tx_validity::{MIN_EXECUTABLE_BALANCE_FOR_INJECTED_MESSAGES, TxValidity, TxValidityChecker}, types::{CommitCertificate, MalachiteEvent}, }; -pub use ethexe_malachite_core::{MalachiteCtx, Multiaddr, ScaleCodec}; +pub use ethexe_malachite_core::{MalachiteConfigEnvironment, MalachiteCtx, Multiaddr, ScaleCodec}; diff --git a/ethexe/malachite/service/src/service.rs b/ethexe/malachite/service/src/service.rs index 211c8012276..d0b21d6e9ba 100644 --- a/ethexe/malachite/service/src/service.rs +++ b/ethexe/malachite/service/src/service.rs @@ -140,12 +140,6 @@ impl MalachiteService { } } - /// Handle a prepared Ethereum block: release pending events whose - /// prerequisite is now satisfied. - pub async fn receive_eb_prepared(&self, _eb_hash: H256) { - self.externalities.drain_pending_events().await - } - /// Tear down the inner consensus core, releasing its store and sockets. pub async fn shutdown(mut self) { if let Some(inner) = self.inner.take() { diff --git a/ethexe/malachite/service/src/starter.rs b/ethexe/malachite/service/src/starter.rs index 75af8cdd1c6..5b192d19b37 100644 --- a/ethexe/malachite/service/src/starter.rs +++ b/ethexe/malachite/service/src/starter.rs @@ -85,6 +85,7 @@ impl MalachiteServiceStarter { }; let core_config = MalachiteCoreConfig { + env: config.env, base: config.home_dir.clone(), validator_secret, validators: config.validators.clone(), @@ -109,7 +110,6 @@ impl MalachiteServiceStarter { }, mempool: mempool.clone(), chain_head: chain_head.clone(), - pending_events: Default::default(), event_tx, }); diff --git a/ethexe/malachite/service/tests/restart_resilience.rs b/ethexe/malachite/service/tests/restart_resilience.rs index 1da31dd2c67..998306ba684 100644 --- a/ethexe/malachite/service/tests/restart_resilience.rs +++ b/ethexe/malachite/service/tests/restart_resilience.rs @@ -135,6 +135,7 @@ fn build_config( pub_key: gsigner::schemes::secp256k1::PublicKey, ) -> MalachiteServiceConfig { MalachiteServiceConfig { + env: Default::default(), gas_allowance: MalachiteServiceConfig::DEFAULT_GAS_ALLOWANCE, canonical_quarantine: 0, post_quarantine_delay: 0, diff --git a/ethexe/network/Cargo.toml b/ethexe/network/Cargo.toml index 0d231b90fbc..b0eecd1bdb5 100644 --- a/ethexe/network/Cargo.toml +++ b/ethexe/network/Cargo.toml @@ -18,27 +18,30 @@ ignored = ["libp2p-gossipsub"] # workspace members gsigner = { workspace = true, features = ["std", "secp256k1", "codec", "keyring", "serde"] } ethexe-db.workspace = true -ethexe-service-utils.workspace = true ethexe-common.workspace = true ethexe-malachite-core.workspace = true gprimitives = { workspace = true, features = ["std", "codec"] } -# libp2p +# libp2p & friends libp2p = { version = "0.56.0", features = ["mdns", "gossipsub", "kad", "identify", "ping", "secp256k1", "request-response", "quic", "tcp", "dns", "tls", "tokio", "macros", "plaintext", "yamux", "metrics"] } libp2p-broadcast.workspace = true libp2p-gossipsub = { version = "0.49.4", features = ["metrics"] } +beetswap = "0.5.0" +blockstore = "0.8.0" +cid = "0.11.3" +multihash = "0.19.5" # other deps tokio = { workspace = true, features = ["macros", "sync"] } anyhow.workspace = true bytes.workspace = true +either = "1.15.0" log.workspace = true parity-scale-codec.workspace = true async-trait.workspace = true rand = { workspace = true, features = ["std", "std_rng"] } futures.workspace = true derive_more.workspace = true -itertools = { workspace = true, features = ["use_std"] } nonempty.workspace = true auto_impl.workspace = true lru.workspace = true diff --git a/ethexe/network/src/bitswap.rs b/ethexe/network/src/bitswap.rs new file mode 100644 index 00000000000..76b8bc9b185 --- /dev/null +++ b/ethexe/network/src/bitswap.rs @@ -0,0 +1,505 @@ +// Copyright (C) Gear Technologies Inc. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +use beetswap::multihasher::{Multihasher, MultihasherError}; +use blockstore::{block::CidError, cond_send::CondSend}; +use cid::{Cid, CidGeneric}; +use ethexe_common::db::HashStorageRO; +use futures::{FutureExt, Stream, stream::FuturesUnordered}; +use gprimitives::H256; +use libp2p::{ + Multiaddr, PeerId, + core::{Endpoint, transport::PortUse}, + swarm::{ + ConnectionDenied, ConnectionId, FromSwarm, NetworkBehaviour, THandler, THandlerInEvent, + THandlerOutEvent, ToSwarm, + }, +}; +use multihash::Multihash; +use std::{ + collections::HashMap, + mem, + sync::Arc, + task::{Context, Poll}, + time::Duration, +}; +use tokio::{ + sync::{mpsc, oneshot}, + task, time, +}; + +const BLAKE2B_CODE: u64 = 0xb220; // standard BLAKE2b multihash code +const RAW_CODEC: u64 = 0x55; // standard CID raw codec + +#[derive(Debug, Clone, Copy, Default)] +pub struct Config { + /// Restart stalled requests after some time. + /// + /// This is intended for test environment only, where a peer + /// can receive an announce request before it has caught up enough to serve + /// the corresponding data. Production environment is expected to have + /// enough peers to fulfill requests, so request scheduling is left to + /// Bitswap itself. + pub auto_retry: bool, +} + +impl Config { + pub fn with_auto_retry(mut self, auto_retry: bool) -> Self { + self.auto_retry = auto_retry; + self + } +} + +#[derive(Clone)] +pub struct Handle { + inner: mpsc::UnboundedSender<(H256, oneshot::Sender>)>, + auto_retry: bool, +} + +impl Handle { + const RETRY_TIMEOUT: Duration = Duration::from_secs(5); + + pub async fn request(&self, request: H256) -> Vec { + if !self.auto_retry { + return self.inner_request(request).await; + } + + loop { + match time::timeout(Self::RETRY_TIMEOUT, self.inner_request(request)).await { + Ok(response) => return response, + Err(_) => { + log::warn!("Bitswap request {request:?} timed out, retrying"); + } + } + } + } + + pub fn request_many<'a>( + &'a self, + iter: impl Iterator + 'a, + ) -> impl Stream)> + 'a { + FuturesUnordered::from_iter( + iter.map(|hash| self.request(hash).map(move |data| (hash, data))), + ) + } + + async fn inner_request(&self, request: H256) -> Vec { + let (tx, rx) = oneshot::channel(); + + self.inner + .send((request, tx)) + .expect("channel should never be closed"); + + rx.await.expect("channel should never be closed") + } +} + +pub(crate) trait BlockstoreDatabase: Send + Sync + HashStorageRO { + fn clone_boxed(&self) -> Box; +} + +impl BlockstoreDatabase for ethexe_db::Database { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } +} + +pub struct Blockstore { + db: Box, +} + +impl Blockstore { + const MAX_BLOCK_SIZE: u64 = 1024 * 1024; // 1MB + + fn convert_multihash(multihash: &Multihash) -> blockstore::Result { + let hash: Multihash<32> = + beetswap::utils::convert_multihash(multihash).ok_or(blockstore::Error::CidTooLarge)?; + if hash.code() != BLAKE2B_CODE { + return Err(blockstore::Error::CidError(CidError::InvalidMultihashCode( + hash.code(), + BLAKE2B_CODE, + ))); + } + if hash.size() as usize != mem::size_of::() { + return Err(blockstore::Error::CidError( + CidError::InvalidMultihashLength(hash.size() as usize), + )); + } + Ok(H256::from_slice(hash.digest())) + } +} + +impl blockstore::Blockstore for Blockstore { + fn get( + &self, + cid: &CidGeneric, + ) -> impl Future>>> + CondSend { + let db = self.db.clone_boxed(); + let hash = *cid.hash(); + let codec = cid.codec(); + task::spawn_blocking(move || { + let hash = Self::convert_multihash(&hash)?; + let data = match codec { + RAW_CODEC => db.read_by_hash(hash), + codec => Err(blockstore::Error::CidError(CidError::InvalidCidCodec( + codec, + )))?, + }; + + if let Some(data) = &data + && data.len() as u64 > Self::MAX_BLOCK_SIZE + { + log::warn!("{hash} is too large: {} bytes", data.len()); + return Err(blockstore::Error::ValueTooLarge); + } + + Ok(data) + }) + .map(|res| { + res.map_err(|err| blockstore::Error::FatalDatabaseError(err.to_string())) + .flatten() + }) + } + + async fn put_keyed( + &self, + _cid: &CidGeneric, + _data: &[u8], + ) -> blockstore::Result<()> { + Ok(()) + } + + async fn remove(&self, _cid: &CidGeneric) -> blockstore::Result<()> { + Ok(()) + } + + async fn close(self) -> blockstore::Result<()> { + Ok(()) + } +} + +struct Blake2b256Multihasher; + +impl Multihasher<32> for Blake2b256Multihasher { + async fn hash( + &self, + multihash_code: u64, + input: &[u8], + ) -> Result, MultihasherError> { + if multihash_code != BLAKE2B_CODE { + return Err(MultihasherError::UnknownMultihashCode); + } + + let hash = ethexe_db::hash(input); + let hash = Multihash::wrap(BLAKE2B_CODE, hash.as_bytes()).expect("size is always correct"); + Ok(hash) + } +} + +type InnerBehaviour = beetswap::Behaviour<32, Blockstore>; + +pub struct Behaviour { + inner: InnerBehaviour, + handle: Handle, + rx: mpsc::UnboundedReceiver<(H256, oneshot::Sender>)>, + requests: HashMap>>, +} + +impl Behaviour { + pub fn new(db: Box, config: Config) -> Self { + let (handle, rx) = mpsc::unbounded_channel(); + let blockstore = Arc::new(Blockstore { db }); + + Self { + inner: InnerBehaviour::builder(blockstore) + .register_multihasher(Blake2b256Multihasher) + .protocol_prefix("/ethexe") + .expect("prefix is always correct") + .build(), + handle: Handle { + inner: handle, + auto_retry: config.auto_retry, + }, + rx, + requests: HashMap::new(), + } + } + + pub fn handle(&self) -> Handle { + self.handle.clone() + } + + fn create_cid(hash: H256) -> Cid { + Cid::new_v1( + RAW_CODEC, + Multihash::wrap(BLAKE2B_CODE, hash.as_bytes()).expect("size is always correct"), + ) + } + + fn handle_inner_event(&mut self, event: beetswap::Event) { + match event { + beetswap::Event::GetQueryResponse { query_id, data } => { + if let Some(channel) = self.requests.remove(&query_id) { + let _ = channel.send(data); + } + } + beetswap::Event::GetQueryError { query_id, error } => { + // The wrapper builds CIDs itself, so invalid multihashes are impossible. + // Blockstore errors here mean local storage violated its read contract. + panic!("{query_id:?} query failed: {error}"); + } + } + } +} + +impl NetworkBehaviour for Behaviour { + type ConnectionHandler = THandler; + type ToSwarm = (); + + fn handle_pending_inbound_connection( + &mut self, + connection_id: ConnectionId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ConnectionDenied> { + self.inner + .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) + } + + fn handle_established_inbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result, ConnectionDenied> { + self.inner.handle_established_inbound_connection( + connection_id, + peer, + local_addr, + remote_addr, + ) + } + + fn handle_pending_outbound_connection( + &mut self, + connection_id: ConnectionId, + maybe_peer: Option, + addresses: &[Multiaddr], + effective_role: Endpoint, + ) -> Result, ConnectionDenied> { + self.inner.handle_pending_outbound_connection( + connection_id, + maybe_peer, + addresses, + effective_role, + ) + } + + fn handle_established_outbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + addr: &Multiaddr, + role_override: Endpoint, + port_use: PortUse, + ) -> Result, ConnectionDenied> { + self.inner.handle_established_outbound_connection( + connection_id, + peer, + addr, + role_override, + port_use, + ) + } + + fn on_swarm_event(&mut self, event: FromSwarm) { + self.inner.on_swarm_event(event); + } + + fn on_connection_handler_event( + &mut self, + peer_id: PeerId, + connection_id: ConnectionId, + event: THandlerOutEvent, + ) { + self.inner + .on_connection_handler_event(peer_id, connection_id, event); + } + + fn poll( + &mut self, + cx: &mut Context<'_>, + ) -> Poll>> { + self.requests.retain(|&query_id, channel| { + if channel.is_closed() { + self.inner.cancel(query_id); + return false; + } + + true + }); + + while let Poll::Ready(Some((request, channel))) = self.rx.poll_recv(cx) { + let cid = Self::create_cid(request); + let query_id = self.inner.get(&cid); + self.requests.insert(query_id, channel); + } + + loop { + match self.inner.poll(cx) { + Poll::Ready(ToSwarm::GenerateEvent(event)) => { + self.handle_inner_event(event); + } + Poll::Ready(to_swarm) => { + return Poll::Ready(to_swarm.map_out(|_event| { + unreachable!("`ToSwarm::GenerateEvent` is handled above") + })); + } + Poll::Pending => return Poll::Pending, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + use blockstore::Blockstore as _; + + #[derive(Clone)] + struct PanickingDatabase; + + impl HashStorageRO for PanickingDatabase { + fn read_by_hash(&self, _hash: H256) -> Option> { + panic!("database read panic"); + } + } + + impl BlockstoreDatabase for PanickingDatabase { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + } + + #[tokio::test] + async fn blockstore_reads_raw_data() { + let db = ethexe_db::Database::memory(); + let hash = db.cas().write(b"hello"); + let blockstore = Blockstore { db: Box::new(db) }; + let cid = Behaviour::create_cid(hash); + + let data = blockstore.get(&cid).await.unwrap(); + + assert_eq!(data, Some(b"hello".to_vec())); + } + + #[tokio::test] + async fn blockstore_rejects_unknown_codec() { + let db = ethexe_db::Database::memory(); + let blockstore = Blockstore { db: Box::new(db) }; + let hash = H256::from([3; 32]); + let multihash = Multihash::wrap(BLAKE2B_CODE, hash.as_bytes()).unwrap(); + let cid = Cid::new_v1(0x99, multihash); + + let error = blockstore.get(&cid).await.unwrap_err(); + + assert_matches!( + error, + blockstore::Error::CidError(CidError::InvalidCidCodec(0x99)) + ); + } + + #[tokio::test] + async fn blockstore_rejects_unknown_multihash_code() { + let db = ethexe_db::Database::memory(); + let blockstore = Blockstore { db: Box::new(db) }; + let hash = H256::from([4; 32]); + let multihash = Multihash::wrap(0x12, hash.as_bytes()).unwrap(); + let cid = Cid::new_v1(RAW_CODEC, multihash); + + let error = blockstore.get(&cid).await.unwrap_err(); + + assert_matches!( + error, + blockstore::Error::CidError(CidError::InvalidMultihashCode(0x12, BLAKE2B_CODE)) + ); + } + + #[tokio::test] + async fn blockstore_rejects_oversized_raw_data() { + let db = ethexe_db::Database::memory(); + let hash = db + .cas() + .write(&vec![0; Blockstore::MAX_BLOCK_SIZE as usize + 1]); + let blockstore = Blockstore { db: Box::new(db) }; + let cid = Behaviour::create_cid(hash); + + let error = blockstore.get(&cid).await.unwrap_err(); + + assert_matches!(error, blockstore::Error::ValueTooLarge); + } + + #[tokio::test] + async fn blockstore_maps_database_panic_to_fatal_database_error() { + let blockstore = Blockstore { + db: Box::new(PanickingDatabase), + }; + let cid = Behaviour::create_cid(H256::from([6; 32])); + + let error = blockstore.get(&cid).await.unwrap_err(); + + assert_matches!( + error, + blockstore::Error::FatalDatabaseError(message) + if message.contains("database read panic") + ); + } + + #[tokio::test] + async fn blake2b_multihasher_hashes_known_code() { + let multihash = Blake2b256Multihasher + .hash(BLAKE2B_CODE, b"hello") + .await + .unwrap(); + + assert_eq!(multihash.code(), BLAKE2B_CODE); + assert_eq!(multihash.digest(), ethexe_db::hash(b"hello").as_bytes()); + } + + #[tokio::test] + async fn blake2b_multihasher_rejects_unknown_code() { + let error = Blake2b256Multihasher + .hash(0x12, b"hello") + .await + .unwrap_err(); + + assert_matches!(error, MultihasherError::UnknownMultihashCode); + } + + #[tokio::test(start_paused = true)] + async fn handle_retries_timed_out_requests() { + let (inner, mut rx) = mpsc::unbounded_channel(); + let handle = Handle { + inner, + auto_retry: true, + }; + let hash = H256::from([5; 32]); + + let pending = tokio::spawn(async move { handle.request(hash).await }); + + let (request, first_response) = rx.recv().await.unwrap(); + assert_eq!(request, hash); + + time::advance(Handle::RETRY_TIMEOUT).await; + let (request, second_response) = rx.recv().await.unwrap(); + assert_eq!(request, hash); + assert!(first_response.is_closed()); + + second_response.send(b"hello".to_vec()).unwrap(); + let response = pending.await.unwrap(); + + assert_eq!(response, b"hello".to_vec()); + } +} diff --git a/ethexe/network/src/db_sync/mod.rs b/ethexe/network/src/db_sync/mod.rs deleted file mode 100644 index 3a3f0d82e37..00000000000 --- a/ethexe/network/src/db_sync/mod.rs +++ /dev/null @@ -1,1179 +0,0 @@ -// Copyright (C) Gear Technologies Inc. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -//! Peer-to-peer database synchronization for `ethexe`. -//! -//! The protocol is built on libp2p request/response and is used to fetch data -//! that can be revalidated locally: raw CAS blobs, program-to-code mappings, -//! and valid code sets. Requests are driven through [`Handle`], while the -//! behaviour internally retries across peers, enforces a per-request -//! timeout, and limits concurrent inbound responses. - -mod requests; -mod responses; - -use crate::{db_sync::requests::OngoingRequests, peer_score, utils::AlternateCollectionFmt}; -pub(crate) use crate::{ - db_sync::{requests::RetriableRequest, responses::OngoingResponses}, - export::{Multiaddr, PeerId}, - utils::ParityScaleCodec, -}; -use async_trait::async_trait; -use ethexe_common::{ - db::{ - BlockMetaStorageRO, CodesStorageRO, ConfigStorageRO, GlobalsStorageRO, HashStorageRO, - MbStorageRO, - }, - gear::CodeState, -}; -use ethexe_db::Database; -use futures::FutureExt; -use gprimitives::{ActorId, CodeId, H256}; -use libp2p::{ - StreamProtocol, - core::{Endpoint, transport::PortUse}, - request_response, - request_response::{InboundFailure, Message, OutboundFailure, ProtocolSupport}, - swarm::{ - ConnectionDenied, ConnectionId, FromSwarm, NetworkBehaviour, THandler, THandlerInEvent, - THandlerOutEvent, ToSwarm, - }, -}; -use parity_scale_codec::{Decode, Encode}; -use std::{ - collections::{BTreeMap, BTreeSet}, - pin::Pin, - sync::atomic::{AtomicU64, Ordering}, - task::{Context, Poll}, - time::Duration, -}; -use tokio::sync::{mpsc, oneshot}; - -const STREAM_PROTOCOL: StreamProtocol = StreamProtocol::new("/ethexe/db-sync/1.0.0"); - -#[derive(Clone, metrics_derive::Metrics)] -#[metrics(scope = "ethexe_network_db_sync")] -struct Metrics { - /// Number of either active or pending requests - ongoing_requests: metrics::Gauge, - /// Number of incoming dropped requests - incoming_dropped_requests: metrics::Counter, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq, derive_more::Display)] -pub enum RequestFailure { - /// Request had been processing for too long - #[display("Request had been processing for too long")] - Timeout, -} - -#[derive(Debug, Eq, PartialEq)] -pub enum Event { - /// Request is in a pending state because there are no peers - NoPeers { - /// The ID of request - request_id: RequestId, - }, - /// Request completion done - RequestSucceed { - /// The ID of request - request_id: RequestId, - }, - /// Request failed - RequestFailed { - /// The failed request - request_id: RequestId, - /// Reason of request failure - error: RequestFailure, - }, - /// Request canceled - /// - /// User dropped [`HandleFuture`]. - /// - /// NOTE: `Event` is not guaranteed in a multithreaded environment - RequestCancelled { - /// The canceled request - request_id: RequestId, - }, - /// Incoming request - IncomingRequest { - /// The ID of in-progress response - response_id: ResponseId, - /// Peer who requested - peer_id: PeerId, - }, - /// Request dropped because simultaneous limit exceeded - IncomingRequestDropped { - /// Peer who should have received the response - peer_id: PeerId, - }, - /// Response sent to incoming request - ResponseSent { - /// The ID of completed response - response_id: ResponseId, - /// Peer who should receive response - peer_id: PeerId, - }, -} - -#[derive(Debug, Clone)] -pub(crate) struct Config { - pub request_timeout: Duration, - pub max_simultaneous_responses: u32, -} - -impl Default for Config { - fn default() -> Self { - Self { - request_timeout: Duration::from_secs(100), - max_simultaneous_responses: 10, - } - } -} - -#[cfg(test)] // used only in tests yet -impl Config { - pub(crate) fn with_request_timeout(mut self, request_timeout: Duration) -> Self { - self.request_timeout = request_timeout; - self - } - - pub(crate) fn with_max_simultaneous_responses( - mut self, - max_simultaneous_responses: u32, - ) -> Self { - self.max_simultaneous_responses = max_simultaneous_responses; - self - } -} - -/// An asynchronous provider of external blockchain data required for response validation. -#[async_trait] -pub trait ExternalDataProvider: Send + Sync { - /// Clone the provider as a trait object. - fn clone_boxed(&self) -> Box; - - /// Resolve program IDs to code IDs at the given block. - async fn programs_code_ids_at( - self: Box, - program_ids: BTreeSet, - block: H256, - ) -> anyhow::Result>; - - /// Resolve code IDs to code states at the given block. - async fn codes_states_at( - self: Box, - code_ids: BTreeSet, - block: H256, - ) -> anyhow::Result>; -} - -#[derive(Debug, Eq, PartialEq, Copy, Clone, Hash)] -pub struct RequestId(u64); - -impl RequestId { - fn next() -> Self { - static COUNTER: AtomicU64 = AtomicU64::new(0); - RequestId(COUNTER.fetch_add(1, Ordering::Relaxed)) - } -} - -#[derive(Debug, Eq, PartialEq, Copy, Clone, Hash)] -pub struct ResponseId(pub(crate) u64); - -#[derive(derive_more::Debug, Default, Clone, Eq, PartialEq, Encode, Decode, derive_more::From)] -pub struct HashesRequest( - #[debug("{:?}", AlternateCollectionFmt::set(_0, "hashes"))] pub BTreeSet, -); - -/// Request to fetch the program-to-code mapping visible at a specific block. -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct ProgramIdsRequest { - pub at: H256, - pub expected_count: u64, -} - -/// Request to fetch the current set of valid codes and verify the response -/// using [`ExternalDataProvider`] at a specific block. -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct ValidCodesRequest { - pub at: H256, - pub validated_count: u64, -} - -/// High-level db-sync request types supported by the network. -#[derive(Debug, Clone, Eq, PartialEq, derive_more::From)] -pub enum Request { - /// Fetch raw CAS blobs by hash. - Hashes(HashesRequest), - /// Fetch the program-to-code mapping for a block. - ProgramIds(ProgramIdsRequest), - /// Fetch the node's locally stored set of valid code IDs. - ValidCodes(ValidCodesRequest), -} - -impl Request { - /// Build a request for a set of CAS hashes. - pub fn hashes(request: impl Into>) -> Self { - Self::Hashes(HashesRequest(request.into())) - } - - /// Build a request for program-to-code mappings at `at`. - pub fn program_ids(at: H256, expected_count: u64) -> Self { - Self::ProgramIds(ProgramIdsRequest { at, expected_count }) - } - - /// Build a request for the valid code set, using `at` only for response - /// verification. - pub fn valid_codes(at: H256, validated_count: u64) -> Self { - Self::ValidCodes(ValidCodesRequest { - at, - validated_count, - }) - } -} - -/// Successful db-sync responses returned to callers. -#[derive(derive_more::Debug, Clone, Eq, PartialEq, derive_more::From, derive_more::Unwrap)] -pub enum Response { - /// Raw CAS blobs keyed by hash. - Hashes(#[debug("{:?}", AlternateCollectionFmt::map(_0, "entries"))] BTreeMap>), - /// Program-to-code mapping reconstructed for a block. - ProgramIds( - #[debug("{:?}", AlternateCollectionFmt::map(_0, "programs"))] BTreeMap, - ), - /// Set of valid code IDs known at a block. - ValidCodes(#[debug("{:?}", AlternateCollectionFmt::set(_0, "codes"))] BTreeSet), -} - -/// Result delivered by [`HandleFuture`]. -pub type HandleResult = Result; - -enum HandleAction { - Request(RequestId, Request), - Retry(RetriableRequest), -} - -impl HandleAction { - fn request_id(&self) -> RequestId { - match self { - HandleAction::Request(request_id, _) => *request_id, - HandleAction::Retry(request) => request.id(), - } - } -} - -/// Future returned by [`Handle::request`] and [`Handle::retry`]. -pub struct HandleFuture { - request_id: RequestId, - rx: oneshot::Receiver, -} - -impl HandleFuture { - /// Returns the identifier assigned to this request. - pub fn request_id(&self) -> RequestId { - self.request_id - } -} - -impl Future for HandleFuture { - type Output = HandleResult; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.rx - .poll_unpin(cx) - .map(|res| res.expect("channel should never be closed")) - } -} - -#[derive(Clone)] -pub struct Handle(mpsc::UnboundedSender<(HandleAction, oneshot::Sender)>); - -impl Handle { - fn send(&self, action: HandleAction) -> HandleFuture { - let (tx, rx) = oneshot::channel(); - let request_id = action.request_id(); - - self.0 - .send((action, tx)) - .expect("channel should never be closed"); - - HandleFuture { request_id, rx } - } - - /// Enqueue a new request. - pub fn request(&self, request: Request) -> HandleFuture { - self.send(HandleAction::Request(RequestId::next(), request)) - } - - /// Re-enqueue a retriable request returned by a previous failure. - pub fn retry(&self, request: RetriableRequest) -> HandleFuture { - self.send(HandleAction::Retry(request)) - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode)] -pub(crate) struct InnerProgramIdsRequest { - at: H256, -} - -/// Network-only type to be encoded-decoded and sent over the network -#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode, derive_more::From)] -pub(crate) enum InnerRequest { - Hashes(HashesRequest), - ProgramIds(InnerProgramIdsRequest), - ValidCodes, -} - -#[derive(Debug, Clone, Default, Eq, PartialEq, Encode, Decode)] -pub(crate) struct InnerHashesResponse(BTreeMap>); - -#[derive(Debug, Default, Eq, PartialEq, Encode, Decode)] -pub(crate) struct InnerProgramIdsResponse(BTreeSet); - -/// Network-only type to be encoded-decoded and sent over the network -#[derive(Debug, Eq, PartialEq, derive_more::From, derive_more::Unwrap, Encode, Decode)] -pub(crate) enum InnerResponse { - Hashes(InnerHashesResponse), - ProgramIds(InnerProgramIdsResponse), - ValidCodes(BTreeSet), -} - -type InnerBehaviour = request_response::Behaviour>; - -#[auto_impl::auto_impl(&, Box)] -pub trait DbSyncDatabase: - Send - + HashStorageRO - + BlockMetaStorageRO - + CodesStorageRO - + ConfigStorageRO - + GlobalsStorageRO - + MbStorageRO -{ - /// Clone the database as a trait object. - fn clone_boxed(&self) -> Box; -} - -impl DbSyncDatabase for Database { - fn clone_boxed(&self) -> Box { - Box::new(self.clone()) - } -} - -pub(crate) struct Behaviour { - inner: InnerBehaviour, - handle: Handle, - rx: mpsc::UnboundedReceiver<(HandleAction, oneshot::Sender)>, - ongoing_requests: OngoingRequests, - ongoing_responses: OngoingResponses, - metrics: Metrics, -} - -impl Behaviour { - pub(crate) fn new( - config: Config, - peer_score_handle: peer_score::Handle, - external_data_provider: Box, - db: Box, - ) -> Self { - let (handle, rx) = mpsc::unbounded_channel(); - let handle = Handle(handle); - - Self { - inner: InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ), - handle, - rx, - ongoing_requests: OngoingRequests::new( - &config, - peer_score_handle, - external_data_provider, - ), - ongoing_responses: OngoingResponses::new(db, &config), - metrics: Metrics::default(), - } - } - - pub fn handle(&self) -> Handle { - self.handle.clone() - } - - fn handle_inner_event( - &mut self, - event: request_response::Event, - ) -> Poll>> { - match event { - request_response::Event::Message { - peer, - connection_id: _, - message: - Message::Request { - request_id: _, - request, - channel, - }, - } => { - let response_id = self - .ongoing_responses - .handle_response(peer, channel, request); - - let event = if let Some(response_id) = response_id { - Event::IncomingRequest { - response_id, - peer_id: peer, - } - } else { - self.metrics.incoming_dropped_requests.increment(1); - Event::IncomingRequestDropped { peer_id: peer } - }; - - return Poll::Ready(ToSwarm::GenerateEvent(event)); - } - request_response::Event::Message { - peer: _, - connection_id: _, - message: - Message::Response { - request_id, - response, - }, - } => { - self.ongoing_requests.on_peer_response(request_id, response); - } - request_response::Event::OutboundFailure { - peer, - connection_id: _, - request_id, - error, - } => { - log::trace!("outbound failure for request {request_id} to {peer}: {error}"); - - if let OutboundFailure::UnsupportedProtocols = error { - log::debug!( - "request to {peer} failed because it doesn't support {STREAM_PROTOCOL} protocol" - ); - } - - self.ongoing_requests.on_peer_failure(request_id); - } - request_response::Event::InboundFailure { - peer, - connection_id: _, - request_id: _, - error: InboundFailure::UnsupportedProtocols, - } => { - log::debug!( - "request from {peer} failed because it doesn't support {STREAM_PROTOCOL} protocol" - ); - } - request_response::Event::InboundFailure { .. } => {} - request_response::Event::ResponseSent { .. } => {} - } - - Poll::Pending - } -} - -impl NetworkBehaviour for Behaviour { - type ConnectionHandler = THandler; - type ToSwarm = Event; - - fn handle_pending_inbound_connection( - &mut self, - connection_id: ConnectionId, - local_addr: &Multiaddr, - remote_addr: &Multiaddr, - ) -> Result<(), ConnectionDenied> { - self.inner - .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) - } - - fn handle_established_inbound_connection( - &mut self, - connection_id: ConnectionId, - peer: PeerId, - local_addr: &Multiaddr, - remote_addr: &Multiaddr, - ) -> Result, ConnectionDenied> { - self.inner.handle_established_inbound_connection( - connection_id, - peer, - local_addr, - remote_addr, - ) - } - - fn handle_pending_outbound_connection( - &mut self, - connection_id: ConnectionId, - maybe_peer: Option, - addresses: &[Multiaddr], - effective_role: Endpoint, - ) -> Result, ConnectionDenied> { - self.inner.handle_pending_outbound_connection( - connection_id, - maybe_peer, - addresses, - effective_role, - ) - } - - fn handle_established_outbound_connection( - &mut self, - connection_id: ConnectionId, - peer: PeerId, - addr: &Multiaddr, - role_override: Endpoint, - port_use: PortUse, - ) -> Result, ConnectionDenied> { - self.inner.handle_established_outbound_connection( - connection_id, - peer, - addr, - role_override, - port_use, - ) - } - - fn on_swarm_event(&mut self, event: FromSwarm) { - self.inner.on_swarm_event(event); - self.ongoing_requests.on_swarm_event(event); - } - - fn on_connection_handler_event( - &mut self, - peer_id: PeerId, - connection_id: ConnectionId, - event: THandlerOutEvent, - ) { - self.inner - .on_connection_handler_event(peer_id, connection_id, event) - } - - fn poll( - &mut self, - cx: &mut Context<'_>, - ) -> Poll>> { - if let Poll::Ready(Some((action, channel))) = self.rx.poll_recv(cx) { - match action { - HandleAction::Request(request_id, request) => { - self.ongoing_requests.request(request_id, request, channel); - } - HandleAction::Retry(request) => { - self.ongoing_requests.retry(request, channel); - } - } - } - - if let Poll::Ready(request_event) = - self.ongoing_requests - .poll(cx, &mut self.inner, &self.metrics) - { - return Poll::Ready(ToSwarm::GenerateEvent(request_event)); - } - - if let Poll::Ready((peer_id, response_id)) = - self.ongoing_responses.poll(cx, &mut self.inner) - { - return Poll::Ready(ToSwarm::GenerateEvent(Event::ResponseSent { - response_id, - peer_id, - })); - } - - if let Poll::Ready(to_swarm) = self.inner.poll(cx) { - return match to_swarm { - ToSwarm::GenerateEvent(event) => self.handle_inner_event(event), - to_swarm => Poll::Ready(to_swarm.map_out::(|_event| { - unreachable!("`ToSwarm::GenerateEvent` is handled above") - })), - }; - } - - Poll::Pending - } -} - -#[cfg(test)] -pub(crate) mod tests { - use super::*; - use crate::{tests::DataProvider, utils::tests::init_logger}; - use assert_matches::assert_matches; - use ethexe_db::Database; - use libp2p::{ - Swarm, Transport, - core::{transport::MemoryTransport, upgrade::Version}, - futures::StreamExt, - identity::Keypair, - swarm, - swarm::SwarmEvent, - }; - use libp2p_swarm_test::SwarmExt; - use std::mem; - use tokio::time; - - // exactly like `Swarm::new_ephemeral_tokio` but we can pass our own config - fn new_ephemeral_swarm( - config: swarm::Config, - behaviour: T, - ) -> Swarm { - let identity = Keypair::generate_ed25519(); - let peer_id = PeerId::from(identity.public()); - - let transport = MemoryTransport::default() - .or_transport(libp2p::tcp::tokio::Transport::default()) - .upgrade(Version::V1) - .authenticate(libp2p::plaintext::Config::new(&identity)) - .multiplex(libp2p::yamux::Config::default()) - .timeout(Duration::from_secs(20)) - .boxed(); - - Swarm::new(transport, behaviour, peer_id, config) - } - - async fn new_swarm_with_config(config: Config) -> (Swarm, Database, DataProvider) { - let data_provider = DataProvider::default(); - let db = Database::memory(); - let behaviour = Behaviour::new( - config, - peer_score::Handle::new_test(), - data_provider.clone_boxed(), - Box::new(db.clone()), - ); - let mut swarm = Swarm::new_ephemeral_tokio(move |_keypair| behaviour); - swarm.listen().with_memory_addr_external().await; - (swarm, db, data_provider) - } - - async fn new_swarm() -> (Swarm, Database, DataProvider) { - new_swarm_with_config(Config::default()).await - } - - #[tokio::test] - async fn smoke() { - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let (mut bob, bob_db, _data_provider) = new_swarm().await; - - let hello_hash = bob_db.cas().write(b"hello"); - let world_hash = bob_db.cas().write(b"world"); - - alice.connect(&mut bob).await; - tokio::spawn(async move { - let mut values = None; - - while let Some(event) = bob.next().await { - let Ok(event) = event.try_into_behaviour_event() else { - continue; - }; - - match event { - Event::IncomingRequest { - response_id, - peer_id, - } => { - values = Some((response_id, peer_id)); - } - Event::ResponseSent { - response_id, - peer_id, - } => { - let (initial_response_id, initial_peer_id) = - values.expect("IncomingRequest must be first"); - assert_eq!(initial_response_id, response_id); - assert_eq!(initial_peer_id, peer_id); - } - _ => {} - } - } - }); - - let request = alice_handle.request(Request::hashes([hello_hash, world_hash])); - let request_id = request.request_id(); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes( - [ - (hello_hash, b"hello".to_vec()), - (world_hash, b"world".to_vec()) - ] - .into() - ) - ) - } - - #[tokio::test(start_paused = true)] - async fn timeout() { - init_logger(); - - let alice_config = Config::default().with_request_timeout(Duration::from_secs(3)); - let (mut alice, _alice_db, _data_provider) = new_swarm_with_config(alice_config).await; - let alice_handle = alice.behaviour().handle(); - - let mut bob = Swarm::new_ephemeral_tokio(|_keypair| { - InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ) - }); - bob.connect(&mut alice).await; - - let request = alice_handle.request(Request::hashes([])); - let request_id = request.request_id(); - - tokio::spawn(async move { - while let Some(event) = bob.next().await { - if let Ok(request_response::Event::Message { - message: - Message::Request { - channel, request, .. - }, - .. - }) = event.try_into_behaviour_event() - { - assert_eq!(request, InnerRequest::Hashes(HashesRequest::default())); - // just ignore request - mem::forget(channel); - } - } - }); - - time::advance(Config::default().request_timeout).await; - - let event = alice.next_behaviour_event().await; - assert_eq!( - event, - Event::RequestFailed { - request_id, - error: RequestFailure::Timeout, - } - ); - request.await.unwrap_err(); - } - - #[tokio::test] - async fn excessive_data_stripped() { - const DATA: [[u8; 1]; 3] = [*b"1", *b"2", *b"3"]; - - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - - let mut bob = Swarm::new_ephemeral_tokio(move |_keypair| { - InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ) - }); - bob.connect(&mut alice).await; - - let data_0 = ethexe_db::hash(&DATA[0]); - let data_1 = ethexe_db::hash(&DATA[1]); - let data_2 = ethexe_db::hash(&DATA[2]); - - let request = alice_handle.request(Request::hashes([data_0, data_1])); - let request_id = request.request_id(); - - tokio::spawn(async move { - while let Some(event) = bob.next().await { - if let Ok(request_response::Event::Message { - message: - Message::Request { - channel, request, .. - }, - .. - }) = event.try_into_behaviour_event() - { - assert_eq!( - request, - InnerRequest::Hashes(HashesRequest([data_0, data_1].into())) - ); - bob.behaviour_mut() - .send_response( - channel, - InnerHashesResponse( - [ - (data_0, DATA[0].to_vec()), - (data_1, DATA[1].to_vec()), - (data_2, DATA[2].to_vec()), - ] - .into(), - ) - .into(), - ) - .unwrap(); - } - } - }); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes([(data_0, DATA[0].to_vec()), (data_1, DATA[1].to_vec())].into()) - ); - } - - #[tokio::test] - async fn truncated_hashes_response_completed_from_same_peer() { - const DATA_LEN: usize = 6 * 1024 * 1024; - - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let (mut bob, bob_db, _data_provider) = new_swarm().await; - - let data_0 = vec![0; DATA_LEN]; - let data_1 = vec![1; DATA_LEN]; - let hash_0 = bob_db.cas().write(&data_0); - let hash_1 = bob_db.cas().write(&data_1); - - alice.connect(&mut bob).await; - tokio::spawn(bob.loop_on_next()); - - let request = alice_handle.request(Request::hashes([hash_0, hash_1])); - let request_id = request.request_id(); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes([(hash_0, data_0), (hash_1, data_1)].into()) - ); - } - - #[tokio::test] - async fn request_response_type_mismatch() { - init_logger(); - - let alice_config = Config::default().with_request_timeout(Duration::ZERO); - let (mut alice, _alice_db, _data_provider) = new_swarm_with_config(alice_config).await; - let alice_handle = alice.behaviour().handle(); - - let mut bob = Swarm::new_ephemeral_tokio(move |_keypair| { - InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ) - }); - bob.connect(&mut alice).await; - - let request = alice_handle.request(Request::hashes([])); - let request_id = request.request_id(); - - tokio::spawn(async move { - while let Some(event) = bob.next().await { - if let Ok(request_response::Event::Message { - message: - Message::Request { - channel, request, .. - }, - .. - }) = event.try_into_behaviour_event() - { - assert_eq!(request, InnerRequest::Hashes(HashesRequest::default())); - bob.behaviour_mut() - .send_response(channel, InnerProgramIdsResponse::default().into()) - .unwrap(); - } - } - }); - - let event = alice.next_behaviour_event().await; - assert_eq!( - event, - Event::RequestFailed { - request_id, - error: RequestFailure::Timeout, - } - ); - - request.await.unwrap_err(); - } - - #[tokio::test] - async fn request_completed_with_3_peers() { - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let (mut bob, bob_db, _data_provider) = new_swarm().await; - let (mut charlie, charlie_db, _data_provider) = new_swarm().await; - let (mut dave, dave_db, _data_provider) = new_swarm().await; - - alice.connect(&mut bob).await; - alice.connect(&mut charlie).await; - alice.connect(&mut dave).await; - tokio::spawn(bob.loop_on_next()); - tokio::spawn(charlie.loop_on_next()); - tokio::spawn(dave.loop_on_next()); - - let hello_hash = bob_db.cas().write(b"hello"); - let world_hash = charlie_db.cas().write(b"world"); - let mark_hash = dave_db.cas().write(b"!"); - - let request = alice_handle.request(Request::hashes([hello_hash, world_hash, mark_hash])); - let request_id = request.request_id(); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes( - [ - (hello_hash, b"hello".to_vec()), - (world_hash, b"world".to_vec()), - (mark_hash, b"!".to_vec()), - ] - .into() - ) - ); - } - - #[tokio::test] - async fn request_completed_after_new_peer() { - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let (mut bob, bob_db, _data_provider) = new_swarm().await; - let (charlie, charlie_db, _data_provider) = new_swarm().await; - let charlie_addr = charlie.external_addresses().next().cloned().unwrap(); - - alice.connect(&mut bob).await; - tokio::spawn(bob.loop_on_next()); - - let hello_hash = bob_db.cas().write(b"hello"); - let world_hash = charlie_db.cas().write(b"world"); - - let request = alice_handle.request(Request::hashes([hello_hash, world_hash])); - let request_id = request.request_id(); - - // first attempt - let event = alice.next_behaviour_event().now_or_never(); - assert_eq!(event, None); - - tokio::spawn(charlie.loop_on_next()); - alice.dial_and_wait(charlie_addr).await; - - // second attempt - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes( - [ - (hello_hash, b"hello".to_vec()), - (world_hash, b"world".to_vec()) - ] - .into() - ) - ); - } - - #[tokio::test(start_paused = true)] - async fn unsupported_protocol_handled() { - const REQUEST_TIMEOUT: Duration = Duration::from_secs(2); - - init_logger(); - - let alice_config = Config::default().with_request_timeout(REQUEST_TIMEOUT); - let (mut alice, _alice_db, _data_provider) = new_swarm_with_config(alice_config).await; - let alice_handle = alice.behaviour().handle(); - - // idle connection timeout is lowered because `libp2p` uses `future_timer` inside, - // so we cannot advance time like in tokio - let mut bob = new_ephemeral_swarm( - swarm::Config::with_tokio_executor() - .with_idle_connection_timeout(Duration::from_secs(5)), - InnerBehaviour::new([], request_response::Config::default()), - ); - let bob_peer_id = *bob.local_peer_id(); - bob.connect(&mut alice).await; - tokio::spawn(bob.loop_on_next()); - - let request = alice_handle.request(Request::hashes([])); - let request_id = request.request_id(); - - // activate timer - let event = alice.next_behaviour_event().now_or_never(); - assert_eq!(event, None); - - time::advance(REQUEST_TIMEOUT).await; - - let event = alice.next_behaviour_event().await; - assert_eq!( - event, - Event::RequestFailed { - request_id, - error: RequestFailure::Timeout - } - ); - - let event = alice.next_swarm_event().await; - assert_matches!(event, SwarmEvent::ConnectionClosed { peer_id, .. } if peer_id == bob_peer_id); - } - - #[tokio::test] - async fn simultaneous_responses_limit() { - init_logger(); - - let alice_config = Config::default().with_max_simultaneous_responses(0); - let (mut alice, _alice_db, _data_provider) = new_swarm_with_config(alice_config).await; - - let (mut bob, _bob_db, _data_provider) = new_swarm().await; - let bob_handle = bob.behaviour().handle(); - let bob_peer_id = *bob.local_peer_id(); - - alice.connect(&mut bob).await; - tokio::spawn(bob.loop_on_next()); - - let fut = bob_handle.request(Request::hashes([])); - mem::forget(fut); - - let event = alice.next_behaviour_event().await; - assert_matches!(event, Event::IncomingRequestDropped { peer_id } if peer_id == bob_peer_id); - } - - #[tokio::test(start_paused = true)] - async fn retry() { - init_logger(); - - let (mut alice, _alice_db, _data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let mut bob = Swarm::new_ephemeral_tokio(move |_keypair| { - InnerBehaviour::new( - [(STREAM_PROTOCOL, ProtocolSupport::Full)], - request_response::Config::default(), - ) - }); - bob.connect(&mut alice).await; - - let request_key = ethexe_db::hash(b"test"); - let request = alice_handle.request(Request::hashes([request_key])); - let request_id = request.request_id(); - - let bob_handle = tokio::spawn(async move { - while let Some(event) = bob.next().await { - if let Ok(request_response::Event::Message { - message: - Message::Request { - channel, request, .. - }, - .. - }) = event.try_into_behaviour_event() - { - assert_eq!( - request, - InnerRequest::Hashes(HashesRequest([request_key].into())) - ); - // just ignore request - mem::forget(channel); - } - } - }); - - time::advance(Config::default().request_timeout).await; - - // first attempt - let event = alice.next_behaviour_event().await; - assert_eq!( - event, - Event::RequestFailed { - request_id, - error: RequestFailure::Timeout, - } - ); - let (error, retriable_request) = request.await.unwrap_err(); - assert_eq!(error, RequestFailure::Timeout); - - time::resume(); - - bob_handle.abort(); - assert!(bob_handle.await.unwrap_err().is_cancelled()); - let (mut charlie, charlie_db, _data_provider) = new_swarm().await; - alice.connect(&mut charlie).await; - tokio::spawn(charlie.loop_on_next()); - - let key = charlie_db.cas().write(b"test"); - assert_eq!(request_key, key); - let request = alice_handle.retry(retriable_request); - let request_id = request.request_id(); - - // retry attempt - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!( - response, - Response::Hashes([(request_key, b"test".to_vec())].into()) - ); - } - - #[tokio::test] - #[ignore = "test setup populates the requester's data provider rather than the responder's; \ - needs a real responder-side fixture"] - async fn external_data_provider() { - init_logger(); - - let (mut alice, _alice_db, alice_data_provider) = new_swarm().await; - let alice_handle = alice.behaviour().handle(); - let (mut bob, _bob_db, _data_provider) = new_swarm().await; - - let expected_response = fill_data_provider(alice_data_provider).await; - - alice.connect(&mut bob).await; - tokio::spawn(bob.loop_on_next()); - - let request = alice_handle.request(Request::program_ids(H256::zero(), 2)); - let request_id = request.request_id(); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestSucceed { request_id }); - - let response = request.await.unwrap(); - assert_eq!(response, expected_response); - } - - #[tokio::test] - async fn request_cancelled() { - let (mut alice, _db, _data_provider) = new_swarm().await; - - let request = alice.behaviour().handle().request(Request::hashes([])); - let request_id = request.request_id(); - drop(request); - - let event = alice.next_behaviour_event().await; - assert_eq!(event, Event::RequestCancelled { request_id }); - } - - pub(crate) async fn fill_data_provider(left_data_provider: DataProvider) -> Response { - let program_ids: BTreeSet = [ActorId::new([1; 32]), ActorId::new([2; 32])].into(); - let code_ids = vec![CodeId::new([0xfe; 32]), CodeId::new([0xef; 32])]; - left_data_provider - .set_programs_code_ids_at(program_ids.clone(), H256::zero(), code_ids.clone()) - .await; - Response::ProgramIds(std::iter::zip(program_ids, code_ids).collect()) - } -} diff --git a/ethexe/network/src/db_sync/requests.rs b/ethexe/network/src/db_sync/requests.rs deleted file mode 100644 index edc17e9c365..00000000000 --- a/ethexe/network/src/db_sync/requests.rs +++ /dev/null @@ -1,831 +0,0 @@ -// Copyright (C) Gear Technologies Inc. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -use crate::{ - db_sync::{ - Config, Event, ExternalDataProvider, HandleResult, HashesRequest, InnerBehaviour, - InnerHashesResponse, InnerProgramIdsRequest, InnerProgramIdsResponse, InnerRequest, - InnerResponse, Metrics, PeerId, ProgramIdsRequest, Request, RequestFailure, RequestId, - Response, ValidCodesRequest, - }, - peer_score::Handle, - utils::{ConnectionMap, NoLimits}, -}; -use anyhow::Context as _; -use ethexe_common::gear::CodeState; -use futures::{FutureExt, future::BoxFuture}; -use gprimitives::{ActorId, CodeId, H256}; -use itertools::EitherOrBoth; -use libp2p::{request_response::OutboundRequestId, swarm::FromSwarm}; -use rand::prelude::IteratorRandom; -use std::{ - cell::OnceCell, - collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}, - iter, - task::{Context, Poll, Waker}, - time::Duration, -}; -use tokio::{sync::oneshot, time}; - -ethexe_service_utils::task_local! { - static CONTEXT: OngoingRequestContext; -} - -type OngoingRequestFuture = BoxFuture<'static, Result>; - -pub(crate) struct OngoingRequests { - pending_events: VecDeque, - requests: HashMap>)>, - active_requests: HashMap, - responses: HashMap>, - connections: ConnectionMap, - waker: Option, - // used in requests themselves - peer_score_handle: Handle, - external_data_provider: Box, - // config - request_timeout: Duration, -} - -impl OngoingRequests { - pub(crate) fn new( - config: &Config, - peer_score_handle: Handle, - external_data_provider: Box, - ) -> Self { - Self { - pending_events: VecDeque::new(), - requests: Default::default(), - active_requests: Default::default(), - responses: Default::default(), - connections: ConnectionMap::without_limits(), - waker: None, - peer_score_handle, - external_data_provider, - request_timeout: config.request_timeout, - } - } - - fn wake(&mut self) { - if let Some(waker) = self.waker.take() { - waker.wake(); - } - } - - /// Tracks all active connections. - pub(crate) fn on_swarm_event(&mut self, event: FromSwarm) { - if self.connections.on_swarm_event(event) { - self.wake(); - } - } - - fn inner_request( - &mut self, - request_id: RequestId, - request: OngoingRequest, - channel: oneshot::Sender, - ) { - self.requests.insert( - request_id, - ( - request - .request( - self.peer_score_handle.clone(), - self.external_data_provider.clone_boxed(), - self.request_timeout, - ) - .boxed(), - Some(channel), - ), - ); - } - - pub(crate) fn request( - &mut self, - request_id: RequestId, - request: Request, - channel: oneshot::Sender, - ) { - self.inner_request(request_id, OngoingRequest::new(request), channel); - } - - pub(crate) fn retry( - &mut self, - request: RetriableRequest, - channel: oneshot::Sender, - ) { - let RetriableRequest { - request_id, - request, - } = request; - self.inner_request(request_id, request, channel); - } - - fn inner_on_peer( - &mut self, - outbound_request_id: OutboundRequestId, - res: Result, - ) { - let request_id = self - .active_requests - .remove(&outbound_request_id) - .expect("unknown outbound request id"); - let fut = self.requests.get_mut(&request_id); - - // request can be removed because of timeout, - // so we don't expect it's still inside `self.requests` - if fut.is_some() { - self.responses.insert(request_id, res); - self.wake(); - } else { - log::trace!("{outbound_request_id:?} has been skipped for {request_id:?}"); - } - } - - pub(crate) fn on_peer_response( - &mut self, - outbound_request_id: OutboundRequestId, - response: InnerResponse, - ) { - self.inner_on_peer(outbound_request_id, Ok(response)); - } - - pub(crate) fn on_peer_failure(&mut self, outbound_request_id: OutboundRequestId) { - self.inner_on_peer(outbound_request_id, Err(())); - } - - pub(crate) fn poll( - &mut self, - cx: &mut Context<'_>, - behaviour: &mut InnerBehaviour, - metrics: &Metrics, - ) -> Poll { - loop { - if let Some(event) = self.pending_events.pop_front() { - return Poll::Ready(event); - } - - let peers: HashSet = self.connections.peers().collect(); - - self.requests.retain(|&request_id, (fut, channel)| { - let response = self.responses.remove(&request_id); - - // it means `HandleFuture` is dropped, - // so we just remove the request and don't make any further work - if channel.as_ref().expect("always Some").is_closed() { - self.pending_events - .push_back(Event::RequestCancelled { request_id }); - return false; - } - - let ctx = OngoingRequestContext { - state: OnceCell::new(), - peers: peers.clone(), - response, - }; - - let (ctx, poll) = CONTEXT.scope(ctx, || fut.poll_unpin(cx)); - let state = ctx.into_state(); - - if let Poll::Ready(res) = poll { - let (event, res) = match res { - Ok(response) => (Event::RequestSucceed { request_id }, Ok(response)), - Err((error, request)) => ( - Event::RequestFailed { request_id, error }, - Err(( - error, - RetriableRequest { - request_id, - request, - }, - )), - ), - }; - - self.pending_events.push_back(event); - - // channel can be dropped after `is_closed()` check during future polling - let res = channel.take().expect("always Some").send(res); - if res.is_err() { - self.pending_events - .push_back(Event::RequestCancelled { request_id }); - } - - return false; - } - - if let Some(state) = state { - match state { - OngoingRequestState::NoPeers => { - self.pending_events.push_back(Event::NoPeers { request_id }); - } - OngoingRequestState::SendRequest(peer, request) => { - let outbound_request_id = behaviour.send_request(&peer, request); - self.active_requests.insert(outbound_request_id, request_id); - } - }; - } - - true - }); - metrics.ongoing_requests.set(self.requests.len() as f64); - - // it means some futures are pending, so we definitely will wake the task - if !self.requests.is_empty() { - self.waker = Some(cx.waker().clone()); - } - - if !self.pending_events.is_empty() { - // immediately return event instead of task waking - continue; - } - - break Poll::Pending; - } - } -} - -#[derive(Debug)] -enum HashesResponseHandled { - Done { - response: BTreeMap>, - stripped: bool, - }, - IncompleteData { - acc: InnerHashesResponse, - new_request: HashesRequest, - stripped: bool, - }, - Err { - acc: InnerHashesResponse, - err: HashesResponseError, - stripped: bool, - }, -} - -impl HashesResponseHandled { - fn stripped(&self) -> bool { - match self { - Self::Done { stripped, .. } => *stripped, - Self::IncompleteData { stripped, .. } => *stripped, - Self::Err { stripped, .. } => *stripped, - } - } -} - -#[derive(Debug, Copy, Clone, Eq, PartialEq, derive_more::Display)] -pub enum HashesResponseError { - #[display("hash mismatch from provided data")] - HashMismatch, -} - -#[derive(Debug, derive_more::Display)] -pub enum ProgramIdsResponseError { - #[display("not enough program-code ids expected {expected} but got {received}")] - NotEnoughIds { expected: usize, received: usize }, - #[display("router failed: {_0}")] - RouterQuery(anyhow::Error), -} - -#[derive(Debug, derive_more::Display)] -pub enum ValidCodesResponseError { - #[display("not enough validated codes")] - NotEnoughCodes, - #[display("{_0}")] - RouterQuery(anyhow::Error), -} - -#[derive(Debug, derive_more::Display, derive_more::From)] -pub(crate) enum ResponseError { - #[display("{_0}")] - Hashes(HashesResponseError), - #[display("{_0}")] - ProgramIds(ProgramIdsResponseError), - #[display("{_0}")] - ValidCodes(ValidCodesResponseError), - #[display("request and response types mismatch")] - TypeMismatch, -} - -#[derive(Debug, derive_more::Unwrap)] -pub(crate) enum ResponseHandlerResult { - Ok(Response), - NewRound(ResponseHandler), - Err(ResponseHandler, ResponseError), -} - -impl From> for ResponseHandlerResult { - fn from(value: Result) -> Self { - match value { - Ok(response) => Self::Ok(response), - Err((handler, error)) => Self::Err(handler, error), - } - } -} - -#[derive(Debug, Eq, PartialEq)] -pub(crate) enum ResponseHandler { - Hashes { - acc: InnerHashesResponse, - request: HashesRequest, - }, - ProgramIds { - request: ProgramIdsRequest, - }, - ValidCodes { - request: ValidCodesRequest, - }, -} - -impl ResponseHandler { - fn new(request: Request) -> Self { - match request { - Request::Hashes(request) => Self::Hashes { - acc: Default::default(), - request, - }, - Request::ProgramIds(request) => Self::ProgramIds { request }, - Request::ValidCodes(request) => Self::ValidCodes { request }, - } - } - - fn inner_request(&self) -> InnerRequest { - match self { - ResponseHandler::Hashes { - request: reduced_request, - .. - } => InnerRequest::Hashes(reduced_request.clone()), - ResponseHandler::ProgramIds { - request: - ProgramIdsRequest { - at, - expected_count: _, - }, - } => InnerRequest::ProgramIds(InnerProgramIdsRequest { at: *at }), - ResponseHandler::ValidCodes { - request: - ValidCodesRequest { - at: _, - validated_count: _, - }, - } => InnerRequest::ValidCodes, - } - } - - fn handle_hashes( - mut acc: InnerHashesResponse, - reduced_request: &HashesRequest, - new_response: InnerHashesResponse, - ) -> HashesResponseHandled { - let mut new_request = BTreeSet::new(); - let mut stripped = false; - - let diff = itertools::merge_join_by( - reduced_request.0.iter().copied(), - new_response.0, - |req_key, (resp_key, _resp_val)| req_key.cmp(resp_key), - ); - - for either in diff { - match either { - EitherOrBoth::Both(req_key, (resp_key, resp_val)) => { - debug_assert_eq!(req_key, resp_key); - if req_key != ethexe_db::hash(&resp_val) { - return HashesResponseHandled::Err { - acc, - err: HashesResponseError::HashMismatch, - stripped, - }; - } - - acc.0.insert(resp_key, resp_val); - } - EitherOrBoth::Left(key) => { - // peer was unable to give this key - new_request.insert(key); - } - EitherOrBoth::Right(_key) => { - // peer sent more keys than we requested - stripped = true; - } - } - } - - if new_request.is_empty() { - HashesResponseHandled::Done { - response: acc.0, - stripped, - } - } else { - HashesResponseHandled::IncompleteData { - acc, - new_request: HashesRequest(new_request), - stripped, - } - } - } - - async fn handle_program_ids( - response: InnerProgramIdsResponse, - request: &ProgramIdsRequest, - external_data_provider: Box, - ) -> Result, ProgramIdsResponseError> { - let InnerProgramIdsResponse(response) = response; - - if response.len() as u64 != request.expected_count { - return Err(ProgramIdsResponseError::NotEnoughIds { - expected: request.expected_count as usize, - received: response.len(), - }); - } - - let code_ids = external_data_provider - .programs_code_ids_at(response.clone(), request.at) - .await - .context("failed to get code ids at block") - .map_err(ProgramIdsResponseError::RouterQuery)?; - - let program_code_ids = iter::zip(response, code_ids).collect(); - Ok(program_code_ids) - } - - async fn handle_valid_codes( - response: BTreeSet, - request: &ValidCodesRequest, - external_data_provider: Box, - ) -> Result, ValidCodesResponseError> { - // validated count at specified block can be less than - // the number of states at the latest block returned by peer - // but cannot be more - if (response.len() as u64) < request.validated_count { - return Err(ValidCodesResponseError::NotEnoughCodes); - } - - let states = external_data_provider - .codes_states_at(response.clone(), request.at) - .await - .context("failed to get code states at block") - .map_err(ValidCodesResponseError::RouterQuery)?; - - let code_ids: BTreeSet = iter::zip(response, states) - .flat_map(|(code_id, state)| { - if state == CodeState::Validated { - Some(code_id) - } else { - None - } - }) - .collect(); - if request.validated_count != code_ids.len() as u64 { - return Err(ValidCodesResponseError::NotEnoughCodes); - } - - Ok(code_ids) - } - - async fn handle( - self, - peer: PeerId, - response: InnerResponse, - peer_score_handle: &Handle, - external_data_provider: Box, - ) -> ResponseHandlerResult { - match (self, response) { - ( - Self::Hashes { - acc, - request: reduced_request, - }, - InnerResponse::Hashes(response), - ) => { - let handled = Self::handle_hashes(acc, &reduced_request, response); - - if handled.stripped() { - log::debug!("data stripped in response from {peer}"); - peer_score_handle.excessive_data(peer); - } - - match handled { - HashesResponseHandled::Done { - response, - stripped: _, - } => ResponseHandlerResult::Ok(Response::Hashes(response)), - HashesResponseHandled::IncompleteData { - acc, - new_request, - stripped: _, - } => ResponseHandlerResult::NewRound(Self::Hashes { - acc, - request: new_request, - }), - HashesResponseHandled::Err { - acc, - err, - stripped: _, - } => ResponseHandlerResult::Err( - Self::Hashes { - acc, - request: reduced_request, - }, - err.into(), - ), - } - } - (Self::ProgramIds { request }, InnerResponse::ProgramIds(response)) => { - Self::handle_program_ids(response, &request, external_data_provider) - .await - .map(Into::into) - .map_err(|err| (Self::ProgramIds { request }, err.into())) - .into() - } - (Self::ValidCodes { request }, InnerResponse::ValidCodes(response)) => { - Self::handle_valid_codes(response, &request, external_data_provider) - .await - .map(Into::into) - .map_err(|err| (Self::ValidCodes { request }, err.into())) - .into() - } - (this, _) => ResponseHandlerResult::Err(this, ResponseError::TypeMismatch), - } - } -} - -#[derive(Debug)] -struct OngoingRequest { - response_handler: Option, - tried_peers: HashSet, -} - -impl OngoingRequest { - fn new(request: Request) -> Self { - Self { - response_handler: Some(ResponseHandler::new(request)), - tried_peers: Default::default(), - } - } - - async fn choose_next_peer(&mut self) -> PeerId { - let mut event_sent = None; - CONTEXT - .poll_fn(|_task_cx, ctx| { - if ctx.peers.is_empty() { - event_sent.get_or_insert_with(|| { - ctx.state - .set(OngoingRequestState::NoPeers) - .expect("set only once"); - }); - return Poll::Pending; - } - - loop { - let peer = ctx - .peers - .difference(&self.tried_peers) - .choose_stable(&mut rand::thread_rng()) - .copied(); - - if let Some(peer) = peer { - self.tried_peers.insert(peer); - break Poll::Ready(peer); - } else { - // just retry all peers again - self.tried_peers.clear(); - continue; - } - } - }) - .await - } - - async fn send_request(&mut self, peer: PeerId) -> Result { - CONTEXT.with_mut(|ctx| { - ctx.state - .set(OngoingRequestState::SendRequest( - peer, - self.response_handler - .as_ref() - .expect("always Some") - .inner_request(), - )) - .expect("set only once"); - }); - - CONTEXT - .poll_fn(|_task_cx, ctx| { - if let Some(res) = ctx.response.take() { - Poll::Ready(res) - } else { - Poll::Pending - } - }) - .await - } - - async fn next_round( - &mut self, - peer_score_handle: &Handle, - external_data_provider: Box, - ) -> Result { - let peer = self.choose_next_peer().await; - - let response = self.send_request(peer).await?; - - match self - .response_handler - .take() - .expect("always Some") - .handle(peer, response, peer_score_handle, external_data_provider) - .await - { - ResponseHandlerResult::Ok(response) => Ok(response), - ResponseHandlerResult::NewRound(handler) => { - log::trace!( - "response is incomplete from peer {peer}: we are going for a new round" - ); - self.response_handler = Some(handler); - Err(()) - } - ResponseHandlerResult::Err(handler, err) => { - log::warn!("response processing failed for request from {peer}: {err:?}"); - peer_score_handle.invalid_data(peer); - self.response_handler = Some(handler); - Err(()) - } - } - } - - async fn request( - mut self, - peer_score_handle: Handle, - external_data_provider: Box, - request_timeout: Duration, - ) -> Result { - let request_loop = async { - loop { - match self - .next_round(&peer_score_handle, external_data_provider.clone_boxed()) - .await - { - Ok(response) => break Ok(response), - Err(()) => continue, - }; - } - }; - - let res = time::timeout(request_timeout, request_loop) - .await - .map_err(|_elapsed| RequestFailure::Timeout) - .and_then(|res| res); - res.map_err(|failure| (failure, self)) - } -} - -#[derive(Debug)] -enum OngoingRequestState { - NoPeers, - SendRequest(PeerId, InnerRequest), -} - -struct OngoingRequestContext { - state: OnceCell, - peers: HashSet, - response: Option>, -} - -impl OngoingRequestContext { - fn into_state(self) -> Option { - let Self { - state, - peers: _, - response, - } = self; - let state = state.into_inner(); - debug_assert_eq!(response, None, "future must take provided response"); - state - } -} - -#[derive(Debug)] -pub struct RetriableRequest { - request_id: RequestId, - request: OngoingRequest, -} - -impl PartialEq for RetriableRequest { - fn eq(&self, other: &Self) -> bool { - self.request_id == other.request_id - } -} - -impl Eq for RetriableRequest {} - -impl RetriableRequest { - pub fn id(&self) -> RequestId { - self.request_id - } -} - -#[cfg(test)] -mod tests { - use super::*; - use async_trait::async_trait; - - struct UnreachableExternalDataProvider; - - #[async_trait] - impl ExternalDataProvider for UnreachableExternalDataProvider { - fn clone_boxed(&self) -> Box { - unreachable!() - } - - async fn programs_code_ids_at( - self: Box, - _program_ids: BTreeSet, - _block: H256, - ) -> anyhow::Result> { - unreachable!() - } - - async fn codes_states_at( - self: Box, - _code_ids: BTreeSet, - _block: H256, - ) -> anyhow::Result> { - unreachable!() - } - } - - #[test] - fn validate_data_stripped() { - let hash1 = ethexe_db::hash(b"1"); - let hash2 = ethexe_db::hash(b"2"); - let hash3 = ethexe_db::hash(b"3"); - - let request = HashesRequest([hash1, hash2].into()); - let response = InnerHashesResponse( - [ - (hash1, b"1".to_vec()), - (hash2, b"2".to_vec()), - (hash3, b"3".to_vec()), - ] - .into(), - ); - let processed = ResponseHandler::handle_hashes(Default::default(), &request, response); - let HashesResponseHandled::Done { response, stripped } = processed else { - unreachable!("{processed:?}") - }; - assert_eq!( - response, - BTreeMap::from_iter([(hash1, b"1".to_vec()), (hash2, b"2".to_vec())]) - ); - assert!(stripped); - } - - #[test] - fn validate_data_hash_mismatch() { - let hash1 = ethexe_db::hash(b"1"); - - let request = HashesRequest([hash1].into()); - let response = InnerHashesResponse([(hash1, b"2".to_vec())].into()); - let processed = ResponseHandler::handle_hashes(Default::default(), &request, response); - let HashesResponseHandled::Err { acc, err, stripped } = processed else { - unreachable!("{processed:?}") - }; - assert_eq!(acc, Default::default()); - assert_eq!(err, HashesResponseError::HashMismatch); - assert!(!stripped); - } - - #[tokio::test] - async fn validate_data_hash_incomplete() { - let hash1 = ethexe_db::hash(b"1"); - let hash2 = ethexe_db::hash(b"2"); - - let request = HashesRequest([hash1, hash2].into()); - let response = InnerHashesResponse([(hash1, b"1".to_vec())].into()); - let processed = - ResponseHandler::handle_hashes(Default::default(), &request, response.clone()); - let HashesResponseHandled::IncompleteData { - acc, - new_request, - stripped, - } = processed - else { - unreachable!("{processed:?}") - }; - assert_eq!(acc, response); - assert_eq!(new_request, HashesRequest([hash2].into())); - assert!(!stripped); - - let handler = ResponseHandler::new(request.into()); - handler - .handle( - PeerId::random(), - response.into(), - &Handle::new_test(), - Box::new(UnreachableExternalDataProvider), - ) - .await - .unwrap_new_round(); - } -} diff --git a/ethexe/network/src/db_sync/responses.rs b/ethexe/network/src/db_sync/responses.rs deleted file mode 100644 index 61de09fadd0..00000000000 --- a/ethexe/network/src/db_sync/responses.rs +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright (C) Gear Technologies Inc. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -use crate::{ - db_sync::{ - Config, DbSyncDatabase, InnerBehaviour, InnerHashesResponse, InnerProgramIdsResponse, - InnerRequest, InnerResponse, ResponseId, - }, - export::PeerId, - utils::ParityScaleCodec, -}; -use libp2p::request_response; -use parity_scale_codec::{Compact, Encode}; -use std::{ - collections::BTreeMap, - task::{Context, Poll}, -}; -use tokio::task::JoinSet; - -struct OngoingResponse { - response_id: ResponseId, - peer_id: PeerId, - channel: request_response::ResponseChannel, - response: InnerResponse, -} - -pub(crate) struct OngoingResponses { - response_id_counter: u64, - db: Box, - db_readers: JoinSet, - max_simultaneous_responses: u32, -} - -impl OngoingResponses { - pub(crate) fn new(db: Box, config: &Config) -> Self { - Self { - response_id_counter: 0, - db, - db_readers: JoinSet::new(), - max_simultaneous_responses: config.max_simultaneous_responses, - } - } - - fn next_response_id(&mut self) -> ResponseId { - let id = self.response_id_counter; - self.response_id_counter += 1; - ResponseId(id) - } - - fn response_from_db(request: InnerRequest, db: Box) -> InnerResponse { - const MAX_RESPONSE_SIZE: u64 = ParityScaleCodec::<(), ()>::MAX_RESPONSE_SIZE; - - match request { - InnerRequest::Hashes(request) => { - let mut response = BTreeMap::new(); - let mut entries_size = 0; - - for hash in request.0 { - let Some(data) = db.read_by_hash(hash) else { - continue; - }; - - let entry_size = hash.encoded_size() + data.encoded_size(); - let next_response_size = 1 // InnerResponse discriminant size - + Compact((response.len() + 1) as u64).encoded_size() - + entries_size - + entry_size; - - if next_response_size > MAX_RESPONSE_SIZE as usize { - // don't try to put other hashes data to prevent abusive database reads - break; - } - - entries_size += entry_size; - response.insert(hash, data); - } - - InnerHashesResponse(response).into() - } - InnerRequest::ProgramIds(request) => { - let actor_ids = match db.mb_program_states(request.at) { - Some(states) => states.into_keys().collect(), - None => { - log::warn!( - "mb_program_states({}) not found; responder returning empty set", - request.at, - ); - Default::default() - } - }; - InnerProgramIdsResponse(actor_ids).into() - } - InnerRequest::ValidCodes => db.valid_codes().into(), - } - } - - pub(crate) fn handle_response( - &mut self, - peer_id: PeerId, - channel: request_response::ResponseChannel, - request: InnerRequest, - ) -> Option { - if self.db_readers.len() >= self.max_simultaneous_responses as usize { - return None; - } - - let response_id = self.next_response_id(); - - let db = self.db.clone_boxed(); - self.db_readers.spawn_blocking(move || { - let response = Self::response_from_db(request, db); - OngoingResponse { - response_id, - peer_id, - channel, - response, - } - }); - - Some(response_id) - } - - pub(crate) fn poll( - &mut self, - cx: &mut Context<'_>, - behaviour: &mut InnerBehaviour, - ) -> Poll<(PeerId, ResponseId)> { - if let Poll::Ready(Some(res)) = self.db_readers.poll_join_next(cx) { - let OngoingResponse { - response_id, - peer_id, - channel, - response, - } = res.expect("database panicked"); - let _res = behaviour.send_response(channel, response); - Poll::Ready((peer_id, response_id)) - } else { - Poll::Pending - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db_sync::HashesRequest; - use ethexe_db::Database; - use gprimitives::H256; - - #[test] - fn response_from_db_truncates_hashes_response_at_encoded_limit() { - const ENTRIES_BEFORE_COMPACT_BOUNDARY: u64 = 0b0011_1111; - const MAX_RESPONSE_SIZE: usize = ParityScaleCodec::<(), ()>::MAX_RESPONSE_SIZE as usize; - - let db = Database::memory(); - - let entries = (0..ENTRIES_BEFORE_COMPACT_BOUNDARY as u8) - .map(|i| vec![i]) - .collect::>(); - let entries_size = entries - .iter() - .map(|data| H256::zero().encoded_size() + data.encoded_size()) - .sum::(); - for data in &entries { - db.cas().write(data); - } - - let last_entry_size = MAX_RESPONSE_SIZE - - 1 // `InnerResponse` discriminant - - Compact(ENTRIES_BEFORE_COMPACT_BOUNDARY + 1).encoded_size() - - entries_size - - H256::zero().encoded_size(); - let last_entry = vec![42; last_entry_size]; - let last_entry_hash = db.cas().write(&last_entry); - - let request = entries - .iter() - .map(|data| ethexe_db::hash(data)) - .chain(Some(last_entry_hash)) - .collect(); - let response = - OngoingResponses::response_from_db(HashesRequest(request).into(), Box::new(db)); - - let response = response.unwrap_hashes(); - assert_eq!(response.0.len(), ENTRIES_BEFORE_COMPACT_BOUNDARY as usize); - assert!(InnerResponse::Hashes(response).encoded_size() <= MAX_RESPONSE_SIZE); - } -} diff --git a/ethexe/network/src/gossipsub.rs b/ethexe/network/src/gossipsub.rs index 54bfcaec365..61ea8fb85b6 100644 --- a/ethexe/network/src/gossipsub.rs +++ b/ethexe/network/src/gossipsub.rs @@ -3,14 +3,12 @@ pub(crate) use libp2p::gossipsub::*; -use crate::{ - db_sync::{Multiaddr, PeerId}, - malachite, peer_score, -}; +use crate::{malachite, peer_score}; use anyhow::anyhow; use bytes::Bytes; use ethexe_common::{Address, injected::SignedCompactTxReceipt, network::SignedValidatorMessage}; use libp2p::{ + Multiaddr, PeerId, core::{Endpoint, transport::PortUse}, gossipsub, identity::Keypair, diff --git a/ethexe/network/src/injected.rs b/ethexe/network/src/injected.rs index 8b3f1c05ee4..2d04aeefbeb 100644 --- a/ethexe/network/src/injected.rs +++ b/ethexe/network/src/injected.rs @@ -1,18 +1,14 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -use crate::{ - db_sync::{Multiaddr, PeerId}, - utils::ParityScaleCodec, - validator::discovery::ValidatorIdentities, -}; +use crate::{utils::ParityScaleCodec, validator::discovery::ValidatorIdentities}; use ethexe_common::{ Address, HashOf, injected::{InjectedTransaction, InjectedTransactionAcceptance, SignedInjectedTransaction}, }; use futures::{FutureExt, StreamExt, future::BoxFuture, stream::FuturesUnordered}; use libp2p::{ - StreamProtocol, + Multiaddr, PeerId, StreamProtocol, core::{Endpoint, transport::PortUse}, request_response, request_response::{ diff --git a/ethexe/network/src/lib.rs b/ethexe/network/src/lib.rs index f0884d24a46..8c778a70776 100644 --- a/ethexe/network/src/lib.rs +++ b/ethexe/network/src/lib.rs @@ -9,16 +9,17 @@ //! - peer management and connection caps; //! - Kademlia-backed validator discovery; //! - gossipsub topics for validator messages and public promises; -//! - request/response database synchronization; +//! - bitswap-backed content fetching; +//! - request/response MB row fetching; //! - private injected-transaction delivery to validators; //! - peer scoring and temporary peer blocking. //! //! [`NetworkService`] is the main integration point used by higher-level //! services. It owns the swarm, emits validated [`NetworkEvent`] items, and -//! hands out protocol-specific handles such as [`db_sync::Handle`] for -//! database synchronization and a peer-scoring handle for internal use. +//! hands out protocol-specific handles such as the bitswap handle for content +//! fetching and a peer-scoring handle for internal use. -pub mod db_sync; +mod bitswap; mod gossipsub; mod injected; mod kad; @@ -36,7 +37,6 @@ pub mod export { pub use injected::Event as NetworkInjectedEvent; use crate::{ - db_sync::DbSyncDatabase, utils::MultiaddrExt, validator::{ValidatorDatabase, list::ValidatorListSnapshot}, }; @@ -179,9 +179,7 @@ pub struct NetworkRuntimeConfig { pub general_signer: Signer, /// Signer used only to construct the libp2p networking keypair. pub network_signer: Signer, - /// External lookups needed by db-sync request validation. - pub external_data_provider: Box, - /// Database backing validator discovery and db-sync responses. + /// Database backing validator discovery and bitswap responses. pub db: Database, } @@ -259,7 +257,6 @@ impl NetworkService { validator_key, general_signer, network_signer, - external_data_provider, db, } = runtime_config; @@ -282,8 +279,7 @@ impl NetworkService { let behaviour_config = BehaviourConfig { router_address, keypair: keypair.clone(), - external_data_provider, - db: DbSyncDatabase::clone_boxed(&db), + db: bitswap::BlockstoreDatabase::clone_boxed(&db), transport_type, validator_key, general_signer, @@ -433,7 +429,7 @@ impl NetworkService { } BehaviourEvent::Kad(event) => self.handle_kad_event(event), BehaviourEvent::Gossipsub(event) => return self.handle_gossipsub_event(event), - BehaviourEvent::DbSync(_event) => {} + BehaviourEvent::Bitswap(_event) => {} BehaviourEvent::Injected(event) => return self.handle_injected_event(event), BehaviourEvent::ValidatorDiscovery(event) => { return self.handle_validator_discovery_event(event); @@ -638,9 +634,9 @@ impl NetworkService { self.swarm.behaviour().peer_score.handle() } - /// Handle used by external services to start db-sync requests. - pub fn db_sync_handle(&self) -> db_sync::Handle { - self.swarm.behaviour().db_sync.handle() + /// Handle used by external services to start Bitswap requests. + pub fn bitswap_handle(&self) -> bitswap::Handle { + self.swarm.behaviour().bitswap.handle() } /// Returns Malachite network parts used by the consensus engine. @@ -712,8 +708,7 @@ impl NetworkService { struct BehaviourConfig<'a> { router_address: Address, keypair: identity::Keypair, - external_data_provider: Box, - db: Box, + db: Box, transport_type: TransportType, validator_key: Option, general_signer: Signer, @@ -743,8 +738,8 @@ pub(crate) struct Behaviour { pub kad: kad::Behaviour, // general communication pub gossipsub: gossipsub::Behaviour, - // database synchronization protocol - pub db_sync: db_sync::Behaviour, + // data request protocol + pub bitswap: bitswap::Behaviour, // injected transaction shenanigans pub injected: injected::Behaviour, // validator discovery @@ -758,7 +753,6 @@ impl Behaviour { let BehaviourConfig { router_address, keypair, - external_data_provider, db, transport_type, validator_key, @@ -818,12 +812,15 @@ impl Behaviour { ) .map_err(|e| anyhow!("`gossipsub::Behaviour` error: {e}"))?; - let db_sync = db_sync::Behaviour::new( - db_sync::Config::default(), - peer_score_handle.clone(), - external_data_provider, - db, - ); + // The test environment can start an announce request while the + // serving peer is still catching up and does not have the announce in + // its local DB yet. Retrying keeps those deterministic integration + // tests from waiting forever; the production environment is expected + // to have enough peers to fulfill requests, so scheduling is left to + // Bitswap itself. + let bitswap = bitswap::Config::default() + .with_auto_retry(matches!(transport_type, TransportType::Test)); + let bitswap = bitswap::Behaviour::new(db, bitswap); let injected = injected::Behaviour::new(); @@ -849,7 +846,7 @@ impl Behaviour { mdns4, kad, gossipsub, - db_sync, + bitswap, injected, validator_discovery, malachite, @@ -860,95 +857,22 @@ impl Behaviour { #[cfg(test)] mod tests { use super::*; - use crate::{ - db_sync::{ExternalDataProvider, tests::fill_data_provider}, - utils::tests::{arb_value, init_logger}, - }; + use crate::utils::tests::{arb_value, init_logger}; use assert_matches::assert_matches; - use async_trait::async_trait; - use ethexe_common::{BlockHeader, ProtocolTimelines, db::*, gear::CodeState}; + use ethexe_common::{BlockHeader, ProtocolTimelines, db::*}; use ethexe_db::Database; - use gprimitives::{ActorId, CodeId, H256}; + use futures::future; + use gprimitives::H256; use gsigner::secp256k1::Signer; use nonempty::nonempty; - use std::{ - collections::{BTreeSet, HashMap}, - future, - num::NonZeroU64, - sync::Arc, - }; + use std::num::NonZeroU64; use tokio::{ - sync::RwLock, time, time::{Duration, timeout}, }; - #[derive(Default)] - struct DataProviderInner { - programs_code_ids_at: HashMap<(BTreeSet, H256), Vec>, - code_states_at: HashMap<(BTreeSet, H256), Vec>, - } - - #[derive(Default, Clone)] - pub struct DataProvider(Arc>); - - impl DataProvider { - pub async fn set_programs_code_ids_at( - &self, - program_ids: BTreeSet, - at: H256, - code_ids: Vec, - ) { - self.0 - .write() - .await - .programs_code_ids_at - .insert((program_ids, at), code_ids); - } - } - - #[async_trait] - impl ExternalDataProvider for DataProvider { - fn clone_boxed(&self) -> Box { - Box::new(self.clone()) - } - - async fn programs_code_ids_at( - self: Box, - program_ids: BTreeSet, - block: H256, - ) -> anyhow::Result> { - assert!(!program_ids.is_empty()); - Ok(self - .0 - .read() - .await - .programs_code_ids_at - .get(&(program_ids, block)) - .cloned() - .unwrap_or_default()) - } - - async fn codes_states_at( - self: Box, - code_ids: BTreeSet, - block: H256, - ) -> anyhow::Result> { - assert!(!code_ids.is_empty()); - Ok(self - .0 - .read() - .await - .code_states_at - .get(&(code_ids, block)) - .cloned() - .unwrap_or_default()) - } - } - struct NetworkServiceBuilder { db: Database, - data_provider: DataProvider, latest_validators: ValidatorsVec, signer: Signer, validator_key: Option, @@ -958,7 +882,6 @@ mod tests { fn new() -> Self { Self { db: Database::memory(), - data_provider: DataProvider::default(), latest_validators: nonempty![Address::default()].into(), signer: Signer::memory(), validator_key: None, @@ -980,7 +903,6 @@ mod tests { let Self { db, - data_provider, latest_validators, signer, validator_key, @@ -1000,7 +922,6 @@ mod tests { validator_key, general_signer: signer.clone(), network_signer: signer, - external_data_provider: Box::new(data_provider), db, }; @@ -1027,7 +948,7 @@ mod tests { init_logger(); let mut service1 = new_service().await; - let service1_handle = service1.db_sync_handle(); + let service1_handle = service1.bitswap_handle(); // second service let service2 = NetworkServiceBuilder::new(); @@ -1041,17 +962,36 @@ mod tests { tokio::spawn(service1.loop_on_next()); tokio::spawn(service2.loop_on_next()); - let request = service1_handle.request(db_sync::Request::hashes([hello, world])); - let response = timeout(Duration::from_secs(5), request) + let hello_request = service1_handle.request(hello); + let world_request = service1_handle.request(world); + let response = timeout( + Duration::from_secs(5), + future::join(hello_request, world_request), + ) + .await + .expect("time has elapsed"); + assert_eq!(response, (b"hello".to_vec(), b"world".to_vec())); + } + + #[tokio::test] + async fn request_db_data_from_connected_peer() { + init_logger(); + + let service2 = NetworkServiceBuilder::new(); + let hello = service2.db.cas().write(b"hello"); + let mut service2 = service2.build().await; + + let mut service1 = NetworkServiceBuilder::new().build().await; + service1.connect(&mut service2).await; + let service1_handle = service1.bitswap_handle(); + + tokio::spawn(service1.loop_on_next()); + tokio::spawn(service2.loop_on_next()); + + let response = timeout(Duration::from_secs(5), service1_handle.request(hello)) .await - .expect("time has elapsed") - .unwrap(); - assert_eq!( - response, - db_sync::Response::Hashes( - [(hello, b"hello".to_vec()), (world, b"world".to_vec())].into() - ) - ); + .expect("time has elapsed"); + assert_eq!(response, b"hello".to_vec()); } #[tokio::test] @@ -1079,34 +1019,6 @@ mod tests { assert_matches!(event, NetworkEvent::PeerBlocked(peer_id) if peer_id == service2_peer_id); } - #[tokio::test] - #[ignore = "test setup populates the requester's data provider rather than the responder's; \ - needs a real responder-side fixture"] - async fn external_data_provider() { - init_logger(); - - let alice = NetworkServiceBuilder::new(); - let alice_data_provider = alice.data_provider.clone(); - let mut alice = alice.build().await; - let alice_handle = alice.db_sync_handle(); - - let bob = NetworkServiceBuilder::new(); - let mut bob = bob.build().await; - - alice.connect(&mut bob).await; - tokio::spawn(alice.loop_on_next()); - tokio::spawn(bob.loop_on_next()); - - let expected_response = fill_data_provider(alice_data_provider).await; - - let request = alice_handle.request(db_sync::Request::program_ids(H256::zero(), 2)); - let response = timeout(Duration::from_secs(5), request) - .await - .expect("time has elapsed") - .unwrap(); - assert_eq!(response, expected_response); - } - #[tokio::test] async fn validator_discovery() { init_logger(); diff --git a/ethexe/network/src/malachite/behaviour.rs b/ethexe/network/src/malachite/behaviour.rs index a31d127be82..25e453e4507 100644 --- a/ethexe/network/src/malachite/behaviour.rs +++ b/ethexe/network/src/malachite/behaviour.rs @@ -1,12 +1,12 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -use crate::{db_sync::Multiaddr, malachite::Config}; +use crate::malachite::Config; use anyhow::anyhow; use bytes::Bytes; -use itertools::Either; +use either::Either; use libp2p::{ - PeerId, + Multiaddr, PeerId, core::{Endpoint, transport::PortUse}, request_response, swarm::{ diff --git a/ethexe/network/src/peer_score.rs b/ethexe/network/src/peer_score.rs index a30691712d9..82089272682 100644 --- a/ethexe/network/src/peer_score.rs +++ b/ethexe/network/src/peer_score.rs @@ -30,14 +30,12 @@ struct Metrics { #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub(crate) enum ScoreDecreaseReason { - ExcessiveData, InvalidData, } impl ScoreDecreaseReason { fn to_i8(self, config: &Config) -> i8 { match self { - ScoreDecreaseReason::ExcessiveData => config.excessive_data, ScoreDecreaseReason::InvalidData => config.invalid_data, } } @@ -55,10 +53,6 @@ impl Handle { Self(tx) } - pub fn excessive_data(&self, peer_id: PeerId) { - let _res = self.0.send((peer_id, ScoreDecreaseReason::ExcessiveData)); - } - pub fn invalid_data(&self, peer_id: PeerId) { let _res = self.0.send((peer_id, ScoreDecreaseReason::InvalidData)); } @@ -83,7 +77,6 @@ pub(crate) enum Event { /// Behaviour config. #[derive(Debug, Clone)] pub(crate) struct Config { - excessive_data: i8, invalid_data: i8, decay: i8, blocked_threshold: i8, @@ -94,7 +87,6 @@ pub(crate) struct Config { impl Config { const fn new() -> Self { Self { - excessive_data: i8::MIN / 5, invalid_data: i8::MIN / 3, decay: i8::MAX / 17, blocked_threshold: i8::MIN / 3, @@ -364,12 +356,12 @@ mod tests { #[tokio::test(start_paused = true)] async fn smoke() { - const EXCESSIVE_DATA: i8 = Config::new().blocked_threshold / 3 - 1; + const INVALID_DATA: i8 = Config::new().blocked_threshold / 3 - 1; init_logger(); let alice_config = Config { - excessive_data: EXCESSIVE_DATA, + invalid_data: INVALID_DATA, ..Default::default() }; let mut alice = new_swarm_with_config(alice_config.clone()).await; @@ -379,37 +371,37 @@ mod tests { tokio::spawn(chad.loop_on_next()); let handle = alice.behaviour_mut().handle(); - handle.excessive_data(chad_peer_id); + handle.invalid_data(chad_peer_id); let event = future::poll_immediate(alice.next_behaviour_event()).await; assert_eq!(event, None); assert_eq!( alice.behaviour().get_score(chad_peer_id), - Some(EXCESSIVE_DATA) + Some(INVALID_DATA) ); - handle.excessive_data(chad_peer_id); + handle.invalid_data(chad_peer_id); let event = future::poll_immediate(alice.next_behaviour_event()).await; assert_eq!(event, None); assert_eq!( alice.behaviour().get_score(chad_peer_id), - Some(2 * EXCESSIVE_DATA) + Some(2 * INVALID_DATA) ); - handle.excessive_data(chad_peer_id); + handle.invalid_data(chad_peer_id); let event = alice.next_behaviour_event().await; assert_eq!( event, Event::PeerBlocked { peer_id: chad_peer_id, - last_reason: ScoreDecreaseReason::ExcessiveData + last_reason: ScoreDecreaseReason::InvalidData } ); assert_eq!( alice.behaviour().get_score(chad_peer_id), - Some(3 * EXCESSIVE_DATA) + Some(3 * INVALID_DATA) ); let event = alice.next_swarm_event().await; @@ -433,7 +425,7 @@ mod tests { ); assert_eq!( alice.behaviour().get_score(chad_peer_id), - Some(EXCESSIVE_DATA * 3 + alice_config.decay) + Some(INVALID_DATA * 3 + alice_config.decay) ); } diff --git a/ethexe/network/src/utils.rs b/ethexe/network/src/utils.rs index a4da1499b28..d11f9a477aa 100644 --- a/ethexe/network/src/utils.rs +++ b/ethexe/network/src/utils.rs @@ -1,17 +1,13 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -use crate::db_sync::PeerId; use async_trait::async_trait; use ip_network::IpNetwork; use libp2p::{ - Multiaddr, StreamProtocol, + Multiaddr, PeerId, StreamProtocol, futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, multiaddr, request_response, - swarm::{ - ConnectionClosed, ConnectionId, DialError, DialFailure, FromSwarm, NewExternalAddrOfPeer, - behaviour::ConnectionEstablished, - }, + swarm::{ConnectionId, DialError, DialFailure, FromSwarm, NewExternalAddrOfPeer}, }; use lru::LruCache; use parity_scale_codec::{Decode, DecodeAll, Encode}; @@ -212,29 +208,6 @@ impl ConnectionMap { limit: NoLimits, } } - - /// Returns true if a new connection added - pub(crate) fn on_swarm_event(&mut self, event: FromSwarm) -> bool { - match event { - FromSwarm::ConnectionEstablished(ConnectionEstablished { - peer_id, - connection_id, - .. - }) => { - let Ok(new) = self.add_connection(peer_id, connection_id); - new - } - FromSwarm::ConnectionClosed(ConnectionClosed { - peer_id, - connection_id, - .. - }) => { - self.remove_connection(peer_id, connection_id); - false - } - _ => false, - } - } } /// A helper struct for formatting collections (BTreeSet, BTreeMap) with two display modes: @@ -401,11 +374,8 @@ impl Default for PeerAddresses { #[cfg(test)] pub(crate) mod tests { - use crate::{ - db_sync::PeerId, - utils::{ConnectionMap, ExponentialBackoffInterval}, - }; - use libp2p::swarm::ConnectionId; + use crate::utils::{ConnectionMap, ExponentialBackoffInterval}; + use libp2p::{PeerId, swarm::ConnectionId}; use proptest::{ arbitrary::Arbitrary, strategy::{Strategy, ValueTree}, diff --git a/ethexe/network/src/validator/discovery.rs b/ethexe/network/src/validator/discovery.rs index 2603e20d551..e398bcac458 100644 --- a/ethexe/network/src/validator/discovery.rs +++ b/ethexe/network/src/validator/discovery.rs @@ -6,7 +6,6 @@ //! Heavily based on Substrate authority discovery mechanism. use crate::{ - db_sync::PeerId, kad::{ self, GetRecordResult, PutRecordFuture, RecordKey, ValidatorIdentityKey, ValidatorIdentityRecord, @@ -27,7 +26,7 @@ use futures::{ use gsigner::secp256k1::{PrivateKey, Secp256k1SignerExt, Signer}; use indexmap::IndexSet; use libp2p::{ - Multiaddr, + Multiaddr, PeerId, core::{Endpoint, transport::PortUse}, identity::Keypair, multiaddr, diff --git a/ethexe/network/src/validator/topic.rs b/ethexe/network/src/validator/topic.rs index 26b20f589ac..a4c1f14391f 100644 --- a/ethexe/network/src/validator/topic.rs +++ b/ethexe/network/src/validator/topic.rs @@ -4,15 +4,13 @@ //! Validator-specific networking logic that verifies signed messages //! against on-chain state. -use crate::{ - db_sync::PeerId, gossipsub::MessageAcceptance, peer_score, - validator::list::ValidatorListSnapshot, -}; +use crate::{gossipsub::MessageAcceptance, peer_score, validator::list::ValidatorListSnapshot}; use ethexe_common::{ Address, HashOf, injected::{InjectedTransaction, SignedCompactTxReceipt}, network::VerifiedValidatorMessage, }; +use libp2p::PeerId; use lru::LruCache; use std::{cmp::Ordering, collections::VecDeque, mem, num::NonZeroUsize, sync::Arc}; diff --git a/ethexe/observer/Cargo.toml b/ethexe/observer/Cargo.toml index d1110db5269..547c30f76f2 100644 --- a/ethexe/observer/Cargo.toml +++ b/ethexe/observer/Cargo.toml @@ -36,7 +36,6 @@ tokio = { workspace = true, features = ["rt-multi-thread", "fs", "sync"] } futures.workspace = true log.workspace = true tracing.workspace = true -derive_more.workspace = true future-timing.workspace = true thiserror.workspace = true gear-workspace-hack.workspace = true diff --git a/ethexe/observer/src/lib.rs b/ethexe/observer/src/lib.rs index 30d3e1c2857..37638904e47 100644 --- a/ethexe/observer/src/lib.rs +++ b/ethexe/observer/src/lib.rs @@ -22,7 +22,6 @@ //! - [`ObserverEvent`] — Stream item: `Block` on a new head, `BlockSynced` after back-fill //! - [`SyncError`] — Error classifier: `RpcError` (recoverable, skipped) vs `Fatal` (propagated) //! - [`utils::BlockLoader`] — Trait abstracting block-data loading from Ethereum -//! - [`utils::BlockId`] — Block selector for `BlockLoader::load_simple`: `Hash(H256)`, `Latest`, `Finalized` //! - [`utils::EthereumBlockLoader`] — alloy-backed [`utils::BlockLoader`] impl //! //! ## Invariants diff --git a/ethexe/observer/src/utils.rs b/ethexe/observer/src/utils.rs index 7b9cdd2367e..d7954937e7a 100644 --- a/ethexe/observer/src/utils.rs +++ b/ethexe/observer/src/utils.rs @@ -17,7 +17,7 @@ use alloy::{ }; use anyhow::{Context, Result}; use ethexe_common::{Address, BlockData, BlockHeader, SimpleBlockData, events::BlockEvent}; -use ethexe_ethereum::{abi::IRouter, mirror, router}; +use ethexe_ethereum::{IntoBlockId, abi::IRouter, mirror, router}; use futures::{TryFutureExt, future}; use gprimitives::H256; use std::{collections::HashMap, future::IntoFuture, ops::RangeInclusive}; @@ -30,26 +30,9 @@ const LOGS_CHUNK_SIZE: u64 = 256; /// Maximum number of in-flight log chunk requests issued by [`alloy::contract::ChunkedEvent`]. const LOGS_MAX_CONCURRENCY: usize = 8; -#[derive(Debug, Copy, Clone, PartialEq, Eq, derive_more::From)] -pub enum BlockId { - Hash(H256), - Latest, - Finalized, -} - -impl BlockId { - fn as_alloy(self) -> alloy::eips::BlockId { - match self { - BlockId::Hash(hash) => alloy::eips::BlockId::hash(hash.0.into()), - BlockId::Latest => alloy::eips::BlockId::latest(), - BlockId::Finalized => alloy::eips::BlockId::finalized(), - } - } -} - #[allow(async_fn_in_trait)] pub trait BlockLoader { - async fn load_simple(&self, block: BlockId) -> Result; + async fn load_simple(&self, block: impl IntoBlockId) -> Result; async fn load(&self, block: H256, header: Option) -> Result; @@ -195,13 +178,10 @@ impl EthereumBlockLoader { } impl BlockLoader for EthereumBlockLoader { - async fn load_simple(&self, block: BlockId) -> Result { + async fn load_simple(&self, block: impl IntoBlockId) -> Result { + let block = block.into_block_id(); log::trace!("Querying simple data for one block {block:?}"); - let block = self - .provider - .get_block(block.as_alloy()) - .into_future() - .await?; + let block = self.provider.get_block(block).into_future().await?; let block = block.context("block not found")?; let (hash, header) = Self::block_response_to_data(block); Ok(SimpleBlockData { hash, header }) @@ -215,7 +195,7 @@ impl BlockLoader for EthereumBlockLoader { let (block_hash, header, logs) = if let Some(header) = header { (block, header, logs_request.await?) } else { - let data = self.load_simple(block.into()); + let data = self.load_simple(block); let (SimpleBlockData { hash, header }, logs) = future::try_join(data, logs_request).await?; (hash, header, logs) diff --git a/ethexe/service/Cargo.toml b/ethexe/service/Cargo.toml index 00eb0b3c584..24509cc3508 100644 --- a/ethexe/service/Cargo.toml +++ b/ethexe/service/Cargo.toml @@ -80,6 +80,7 @@ demo-async-init = { workspace = true, features = ["debug", "ethexe"] } demo-piggy-bank = { workspace = true, features = ["debug", "ethexe"] } demo-reply-callback = { workspace = true, features = ["debug", "ethexe"] } demo-fungible-token = { workspace = true } +demo-mul-by-const = { workspace = true } demo-delayed-sender-ethexe = { workspace = true, features = [ "debug", "ethexe", diff --git a/ethexe/service/src/fast_sync.rs b/ethexe/service/src/fast_sync.rs index 622a890f0cb..375e6aa578a 100644 --- a/ethexe/service/src/fast_sync.rs +++ b/ethexe/service/src/fast_sync.rs @@ -1,16 +1,1128 @@ // Copyright (C) Gear Technologies Inc. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -//! Fast synchronization stub — always fails so `--fast-sync` doesn't -//! silently degrade to a full-from-genesis catch-up. +//! Fast synchronization for MB-driven ethexe nodes. use crate::Service; -use anyhow::{Result, bail}; +use alloy::eips::BlockId; +use anyhow::{Context, Result, anyhow}; +use ethexe_common::{ + Address, BlockData, CodeAndIdUnchecked, Digest, ProgramStates, StateHashWithQueueSize, + db::{ + BlockMetaStorageRO, CodesStorageRO, CodesStorageRW, ConfigStorageRO, GlobalsStorageRW, + MbStorageRO, MbStorageRW, OnChainStorageRW, PreparedBlockData, + }, + events::{ + BlockEvent, RouterEvent, + router::{BatchCommittedEvent, EBCommittedEvent, MBCommittedEvent}, + }, + injected, +}; +use ethexe_compute::ComputeService; +use ethexe_db::{ + Database, + iterator::{ + DatabaseIteratorError, DatabaseIteratorStorage, DispatchStashNode, MailboxNode, + MemoryPagesNode, MemoryPagesRegionNode, MessageQueueNode, ProgramStateNode, + UserMailboxNode, WaitlistNode, + }, + visitor::DatabaseVisitor, +}; +use ethexe_ethereum::{mirror::MirrorQuery, router::RouterQuery}; +use ethexe_network::NetworkService; +use ethexe_observer::{ObserverService, utils::BlockLoader}; +use ethexe_runtime_common::{ + ScheduleRestorer, + state::{ + DispatchStash, Mailbox, MemoryPages, MemoryPagesRegion, MessageQueue, ProgramState, + UserMailbox, Waitlist, + }, +}; +use futures::StreamExt; +use gprimitives::{ActorId, CodeId, H256}; +use parity_scale_codec::Decode; +use std::{ + cmp::max, + collections::{BTreeMap, BTreeSet, HashMap}, +}; -// TODO: #5487 implement the actual fast-sync logic. -pub(crate) async fn sync(_service: &mut Service) -> Result<()> { - bail!( - "fast-sync is not implemented for the MB-driven recovery path yet; \ - start the node without --fast-sync (or omit `fast_sync = true` in config)" +/// The on-chain committed anchor fast sync restores the database to: the +/// latest committed MB (`mb_hash`) and the Ethereum block it advanced to +/// (`eb_hash`). The service uses `mb_hash` as the boundary for its startup +/// replay gate — Malachite events up to and including this MB are suppressed +/// (already restored here), and live consensus resumes past it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FastSyncReplayTarget { + pub mb_hash: H256, + pub eb_hash: H256, +} + +/// Startup gate that suppresses Malachite's replay of the already-restored +/// chain after fast sync. +/// +/// When Malachite's core starts it value-syncs from genesis and replays every +/// historical MB as a `BlockProposal` (which is what drives compute; the +/// matching `BlockFinalized` always follows and does no compute). Fast sync +/// only restored the DB up to `latest_committed_mb`, so these replayed +/// proposals must not drive compute. Malachite emits blocks in height order, so +/// it suffices to drop proposals until the target MB is reached. Two cases: +/// * fresh DB — Malachite replays from genesis; we wait for the target by hash; +/// * target already finalized locally before this start — Malachite resumes +/// from the target's child, so the target hash never reappears. The first +/// proposal is then a descendant of the target; an ancestry walk detects that +/// and opens the gate immediately. +/// +/// The ancestry walk runs at most once (the first proposal); afterwards the +/// decision is a single hash comparison. +pub(crate) enum ReplayGate { + /// No fast sync this run — never gates. + Open, + /// Awaiting the first proposal to disambiguate the two cases above. + Pending(H256), + /// Genesis replay confirmed; waiting for the target MB by hash. + Waiting(H256), +} + +impl ReplayGate { + pub(crate) fn new(target: Option) -> Self { + match target { + Some(target) => Self::Pending(target.mb_hash), + None => Self::Open, + } + } + + /// Whether the `BlockProposal` for `mb_hash` should drive compute; opens + /// the gate once the target boundary is reached. Only proposals are gated + /// because compute is triggered solely by `BlockProposal` (the matching + /// `BlockFinalized` always follows and does no compute). + pub(crate) fn allow(&mut self, db: &Database, mb_hash: H256) -> Result { + match *self { + Self::Open => Ok(true), + Self::Pending(target) => { + if mb_hash == target { + // The restored boundary itself — already computed by fast sync. + *self = Self::Open; + Ok(false) + } else if target_is_ancestor(db, target, mb_hash)? { + // Target was finalized before this start; this is the first + // live descendant — process it and resume normally. + *self = Self::Open; + Ok(true) + } else { + *self = Self::Waiting(target); + Ok(false) + } + } + Self::Waiting(target) => { + if mb_hash == target { + *self = Self::Open; + } + Ok(false) + } + } + } +} + +/// Walk the MB parent chain from `mb_hash` and report whether `target` is a +/// strict ancestor. Errors loudly on a missing `CompactMb` rather than hanging +/// the gate on incomplete data. +fn target_is_ancestor(db: &Database, target: H256, mb_hash: H256) -> Result { + let mut current = mb_hash; + loop { + let parent = db + .mb_compact_block(current) + .ok_or_else(|| { + anyhow!( + "fast-sync replay gate: missing CompactMb for {current} while resolving \ + ancestry of finalized MB {mb_hash}" + ) + })? + .parent; + if parent == target { + return Ok(true); + } + if parent.is_zero() { + return Ok(false); + } + current = parent; + } +} + +struct EventData { + latest_committed_batch: Digest, + replay_target: FastSyncReplayTarget, +} + +impl EventData { + async fn collect( + block_loader: &impl BlockLoader, + db: &Database, + highest_block: H256, + ) -> Result> { + let mut block = highest_block; + while !db.block_meta(block).prepared { + let block_data = block_loader.load(block, None).await?; + + let mut latest_committed_batch = None; + let mut pending_eb = None; + for event in block_data.events.iter().rev() { + match event { + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest, + })) => { + latest_committed_batch.get_or_insert(*digest); + } + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(eb_hash))) => { + pending_eb = Some(*eb_hash); + } + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb_hash))) => { + if let Some(eb_hash) = pending_eb.take() { + return Ok(Some(Self { + // Keep the latest batch digest from this same + // EB block; later batch-only commits in the + // block still advance Router's previous-batch + // state for the next submission. + latest_committed_batch: latest_committed_batch.context( + "committed MB replay target without BatchCommitted event", + )?, + replay_target: FastSyncReplayTarget { + mb_hash: *mb_hash, + eb_hash, + }, + })); + } + } + _ => {} + } + } + + block = block_data.header.parent_hash; + if block.is_zero() { + break; + } + } + + Ok(None) + } +} + +async fn collect_program_code_ids( + observer: &mut ObserverService, + genesis_block: u32, + latest_committed_block: u32, +) -> Result> { + Ok(observer + .router_query() + .events() + .program_created() + .from_block(genesis_block) + .to_block(latest_committed_block) + .query() + .await + .context("failed to query ProgramCreated events")? + .into_iter() + .map(|(event, _log)| (event.actor_id, event.code_id)) + .collect()) +} + +async fn collect_program_state_hashes( + observer: &mut ObserverService, + block_hash: H256, + program_code_ids: &BTreeMap, +) -> Result> { + let mut program_states = BTreeMap::new(); + let provider = observer.provider(); + + for &actor_id in program_code_ids.keys() { + let mirror = Address::try_from(actor_id).expect("invalid actor id"); + let mirror = MirrorQuery::new(provider.clone(), mirror); + + let state_hash = mirror.state_hash_at(block_hash).await.with_context(|| { + format!("failed to get state hash for actor {actor_id} at block {block_hash}") + })?; + + if state_hash.is_zero() { + log::trace!( + "skipping program with zero mirror state during fast sync: actor {actor_id}, block {block_hash}" + ); + continue; + } + + program_states.insert(actor_id, state_hash); + } + + Ok(program_states) +} + +async fn collect_code_ids( + observer: &mut ObserverService, + genesis_block: u32, + latest_committed_block: u32, +) -> Result> { + Ok(observer + .router_query() + .events() + .code_got_validated() + .valid(true) + .from_block(genesis_block) + .to_block(latest_committed_block) + .query() + .await + .context("failed to query CodeGotValidated events")? + .into_iter() + .map(|(event, _log)| event.code_id) + .collect()) +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RequestMetadata { + ProgramState, + MemoryPages, + MemoryPagesRegion, + MessageQueue, + Waitlist, + Mailbox, + UserMailbox, + DispatchStash, + /// Any data we only insert into the database. + Data, +} + +impl RequestMetadata { + fn is_data(self) -> bool { + matches!(self, RequestMetadata::Data) + } +} + +#[derive(Debug)] +struct RequestManager { + db: Database, + + /// Total completed requests + total_completed_requests: u64, + /// Total pending requests + total_pending_requests: u64, + + /// Pending requests are either: + /// * Skipped if they are `RequestMetadata::Data` and exist in the database + /// * Completed if the database has keys + /// * Converted into one network request + pending_requests: HashMap, + /// Completed requests + responses: Vec<(RequestMetadata, Vec)>, +} + +impl RequestManager { + fn new(db: Database) -> Self { + Self { + db, + total_completed_requests: 0, + total_pending_requests: 0, + pending_requests: HashMap::new(), + responses: Vec::new(), + } + } + + fn add(&mut self, hash: H256, metadata: RequestMetadata) { + assert_ne!( + hash, + H256::zero(), + "zero hash is cannot be requested from db or network" + ); + + let old_metadata = self.pending_requests.insert(hash, metadata); + + if let Some(old_metadata) = old_metadata { + debug_assert_eq!(metadata, old_metadata); + } else { + self.total_pending_requests += 1; + } + } + + async fn request( + &mut self, + network: &mut NetworkService, + ) -> Option)>> { + let pending_network_requests = self.handle_pending_requests(); + + if !pending_network_requests.is_empty() { + let bitswap = network.bitswap_handle(); + let mut request = bitswap + .request_many(pending_network_requests.keys().copied()) + .collect(); + + let response = loop { + tokio::select! { + _ = network.select_next_some() => {}, + response = &mut request => break response, + } + }; + drop(request); + + self.handle_response(pending_network_requests, response); + } + + let continue_processing = !(self.pending_requests.is_empty() && self.responses.is_empty()); + if continue_processing { + let responses: Vec<_> = self.responses.drain(..).collect(); + self.total_completed_requests += responses.len() as u64; + Some(responses) + } else { + None + } + } + + fn handle_pending_requests(&mut self) -> HashMap { + let mut pending_requests = HashMap::new(); + for (hash, metadata) in self.pending_requests.drain() { + if metadata.is_data() && self.db.cas().contains(hash) { + self.total_completed_requests += 1; + continue; + } + + if let Some(data) = self.db.cas().read(hash) { + self.responses.push((metadata, data)); + continue; + } + + pending_requests.insert(hash, metadata); + } + + pending_requests + } + + fn handle_response( + &mut self, + mut pending_network_requests: HashMap, + data: Vec<(H256, Vec)>, + ) { + for (hash, data) in data { + let metadata = pending_network_requests + .remove(&hash) + .expect("unknown pending request"); + + let db_hash = self.db.cas().write(&data); + debug_assert_eq!(hash, db_hash); + + self.responses.push((metadata, data)); + } + + debug_assert_eq!( + pending_network_requests, + HashMap::new(), + "network service guarantees it gathers all hashes" + ); + } + + /// (total completed request, total pending requests) + fn stats(&self) -> (u64, u64) { + let completed = self.total_completed_requests; + let pending = self.total_pending_requests; + debug_assert!(completed <= pending, "{completed} <= {pending}"); + (completed, pending) + } +} + +impl DatabaseVisitor for RequestManager { + fn db(&self) -> &dyn DatabaseIteratorStorage { + &self.db + } + + fn clone_boxed_db(&self) -> Box { + Box::new(self.db.clone()) + } + + fn on_db_error(&mut self, error: DatabaseIteratorError) { + let (hash, metadata) = match error { + DatabaseIteratorError::NoMemoryPages(hash) => { + (hash.inner(), RequestMetadata::MemoryPages) + } + DatabaseIteratorError::NoMemoryPagesRegion(hash) => { + (hash.inner(), RequestMetadata::MemoryPagesRegion) + } + DatabaseIteratorError::NoPageData(hash) => (hash.inner(), RequestMetadata::Data), + DatabaseIteratorError::NoMessageQueue(hash) => { + (hash.inner(), RequestMetadata::MessageQueue) + } + DatabaseIteratorError::NoWaitlist(hash) => (hash.inner(), RequestMetadata::Waitlist), + DatabaseIteratorError::NoDispatchStash(hash) => { + (hash.inner(), RequestMetadata::DispatchStash) + } + DatabaseIteratorError::NoMailbox(hash) => (hash.inner(), RequestMetadata::Mailbox), + DatabaseIteratorError::NoUserMailbox(hash) => { + (hash.inner(), RequestMetadata::UserMailbox) + } + DatabaseIteratorError::NoAllocations(hash) => (hash.inner(), RequestMetadata::Data), + DatabaseIteratorError::NoProgramState(hash) => (hash, RequestMetadata::ProgramState), + DatabaseIteratorError::NoPayload(hash) => (hash.inner(), RequestMetadata::Data), + DatabaseIteratorError::NoOriginalCode(code_id) => { + (code_id.into(), RequestMetadata::Data) + } + DatabaseIteratorError::NoCodeValid(_) + | DatabaseIteratorError::NoInstrumentedCode(_) + | DatabaseIteratorError::NoCodeMetadata(_) => return, + DatabaseIteratorError::NoBlockHeader(_) + | DatabaseIteratorError::NoBlockEvents(_) + | DatabaseIteratorError::NoBlockCodesQueue(_) + | DatabaseIteratorError::NoMb(_) + | DatabaseIteratorError::NoMbProgramStates(_) + | DatabaseIteratorError::NoMbSchedule(_) + | DatabaseIteratorError::NoMbOutcome(_) + | DatabaseIteratorError::NoProgramCodeId(_) => { + unreachable!("{error:?}") + } + }; + self.add(hash, metadata); + } +} + +impl Drop for RequestManager { + fn drop(&mut self) { + #[cfg(debug_assertions)] + { + let Self { + db: _, + total_completed_requests, + total_pending_requests, + pending_requests, + responses, + } = self; + assert_eq!(total_completed_requests, total_pending_requests); + assert_eq!(*pending_requests, HashMap::new()); + assert_eq!(*responses, Vec::new()); + } + } +} + +/// Synchronize program states and related data from the network. +/// +/// This asynchronous function fetches data from the network based on program +/// state hashes and associated metadata using a request-manager mechanism. It also enriches +/// the program states with cached queue sizes. +async fn sync_from_network( + network: &mut NetworkService, + db: &Database, + code_ids: &BTreeSet, + program_states: BTreeMap, +) -> ProgramStates { + let mut restored_cached_queue_sizes = BTreeMap::new(); + + let mut manager = RequestManager::new(db.clone()); + + for &state in program_states.values() { + manager.add(state, RequestMetadata::ProgramState); + } + + for &code_id in code_ids { + manager.add(code_id.into(), RequestMetadata::Data); + } + + loop { + let (completed, pending) = manager.stats(); + log::info!("[{completed:>05} / {pending:>05}] Getting network data"); + + let Some(responses) = manager.request(network).await else { + break; + }; + + for (metadata, data) in responses { + match metadata { + RequestMetadata::ProgramState => { + let state: ProgramState = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + let program_state_hash = ethexe_db::hash(&data); + restored_cached_queue_sizes.insert( + program_state_hash, + ( + state.canonical_queue.cached_queue_size, + state.injected_queue.cached_queue_size, + ), + ); + ethexe_db::visitor::walk( + &mut manager, + ProgramStateNode { + program_state: state, + }, + ); + } + RequestMetadata::MemoryPages => { + let memory_pages: MemoryPages = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, MemoryPagesNode { memory_pages }); + } + RequestMetadata::MemoryPagesRegion => { + let memory_pages_region: MemoryPagesRegion = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk( + &mut manager, + MemoryPagesRegionNode { + memory_pages_region, + }, + ); + } + RequestMetadata::MessageQueue => { + let message_queue: MessageQueue = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, MessageQueueNode { message_queue }); + } + RequestMetadata::Waitlist => { + let waitlist: Waitlist = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, WaitlistNode { waitlist }); + } + RequestMetadata::Mailbox => { + let mailbox: Mailbox = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, MailboxNode { mailbox }); + } + RequestMetadata::UserMailbox => { + let user_mailbox: UserMailbox = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, UserMailboxNode { user_mailbox }); + } + RequestMetadata::DispatchStash => { + let dispatch_stash: DispatchStash = + Decode::decode(&mut &data[..]).expect("bitswap must validate data"); + ethexe_db::visitor::walk(&mut manager, DispatchStashNode { dispatch_stash }); + } + RequestMetadata::Data => continue, + } + } + } + + log::info!("Network data getting is done"); + + // Enrich program states with cached queue size + program_states + .into_iter() + .map(|(program_id, hash)| { + let (canonical_queue_size, injected_queue_size) = *restored_cached_queue_sizes + .get(&hash) + .expect("program state cached queue size must be restored"); + ( + program_id, + StateHashWithQueueSize { + hash, + canonical_queue_size, + injected_queue_size, + }, + ) + }) + .collect() +} + +/// Instruments a set of codes by delegating their processing to the `ComputeService`. +async fn instrument_codes( + compute: &mut ComputeService, + db: &Database, + mut code_ids: BTreeSet, +) -> Result<()> { + // `code_valid` is compute's processed marker; valid codes must have instrumented bytes. + code_ids.retain(|&code_id| db.code_valid(code_id).is_none()); + + if code_ids.is_empty() { + log::info!("No codes to instrument. Skipping..."); + return Ok(()); + } + + log::info!("Instrument {} codes", code_ids.len()); + + for &code_id in &code_ids { + let original_code = db + .original_code(code_id) + .expect("`sync_from_network` must fulfill database"); + compute.process_code(CodeAndIdUnchecked { + code_id, + code: original_code, + }); + } + + while let Some(event) = compute.next().await { + let id = event?.unwrap_code_processed(); + code_ids.remove(&id); + if code_ids.is_empty() { + break; + } + } + + log::info!("Codes instrumentation done"); + Ok(()) +} + +async fn set_tx_pool_data_requirement( + db: &Database, + block_loader: &impl BlockLoader, + latest_committed_block_height: u32, +) -> Result<()> { + let to = latest_committed_block_height as u64; + let from = to.saturating_sub(injected::VALIDITY_WINDOW as u64); + + // TODO: #4926 unsafe solution - we need it for taking events from predecessor blocks in ethexe-compute + let blocks = block_loader.load_many(from..=to).await?; + for BlockData { + hash, + header, + events, + } in blocks.into_values() + { + db.set_block_header(hash, header); + db.set_block_events(hash, &events); + } + + Ok(()) +} + +async fn latest_era_with_committed_validators( + db: &Database, + router: &RouterQuery, + block_hash: H256, +) -> Result { + let storage_view = router.storage_view_at(block_hash).await?; + + db.config() + .timelines + .era_from_ts(max( + storage_view + .validationSettings + .validators0 + .useFromTimestamp + .to::(), + storage_view + .validationSettings + .validators1 + .useFromTimestamp + .to::(), + )) + .context("failed to calculate era from validators timestamp") +} + +pub(crate) async fn sync(service: &mut Service) -> Result> { + let Service { + observer, + compute, + network, + db, + #[cfg(test)] + sender, + .. + } = service; + log::info!("Fast synchronization is in progress..."); + + let finalized_block = observer + .block_loader() + // we get finalized block to avoid block reorganization + // because we restore the database only for the latest block of a chain, + // and thus the reorganization can lead us to an empty block + .load_simple(BlockId::finalized()) + .await + .context("failed to get latest finalized block")? + .hash; + + let genesis_block_hash = db.config().genesis_block_hash; + let genesis_block = observer + .block_loader() + .load_simple(genesis_block_hash) + .await + .context("failed to get genesis block")?; + + let block_loader = observer.block_loader(); + let Some(event_data) = EventData::collect(&block_loader, db, finalized_block).await? else { + log::warn!("No finalized committed MB found. Skipping fast synchronization..."); + return Ok(None); + }; + + let EventData { + latest_committed_batch, + replay_target, + } = event_data; + + let FastSyncReplayTarget { + mb_hash: latest_committed_mb, + eb_hash: latest_committed_eb, + } = replay_target; + let block_data = block_loader.load(latest_committed_eb, None).await?; + + let code_ids = collect_code_ids( + observer, + genesis_block.header.height, + block_data.header.height, + ) + .await?; + let program_code_ids = collect_program_code_ids( + observer, + genesis_block.header.height, + block_data.header.height, + ) + .await?; + let program_state_hashes = + collect_program_state_hashes(observer, finalized_block, &program_code_ids).await?; + + let program_states = sync_from_network(network, db, &code_ids, program_state_hashes).await; + + instrument_codes(compute, db, code_ids).await?; + + let schedule = ScheduleRestorer::from_storage(db, &program_states)?.restore(); + + set_tx_pool_data_requirement(db, &block_loader, block_data.header.height).await?; + + for (program_id, code_id) in program_code_ids { + db.set_program_code_id(program_id, code_id); + } + + let latest_era_with_committed_validators = + latest_era_with_committed_validators(db, &observer.router_query(), latest_committed_eb) + .await?; + + ethexe_common::setup_block_in_db( + db, + latest_committed_eb, + PreparedBlockData { + header: block_data.header, + events: block_data.events.clone(), + latest_era_with_committed_validators, + // NOTE: there is no invariant that fast sync should recover codes queue + codes_queue: Default::default(), + last_committed_batch: latest_committed_batch, + last_committed_mb: latest_committed_mb, + last_committed_eb: latest_committed_eb, + }, + ); + + db.mutate_mb_meta(latest_committed_mb, |meta| { + meta.computed = true; + meta.last_advanced_eb = latest_committed_eb; + }); + + // NOTE: Malachite should restore `CompactMb` by itself + //db.set_mb_compact_block(latest_committed_mb, compact); + + db.set_mb_program_states(latest_committed_mb, program_states.clone()); + db.set_mb_schedule(latest_committed_mb, schedule); + // The committed fast-sync anchor is never replayed for batch creation, so + // keep queue-derived outcome invariants out of restoration and persist an + // empty row only to satisfy database walks over committed MB metadata. + db.set_mb_outcome(latest_committed_mb, Default::default()); + + db.globals_mutate(|globals| { + globals.start_block_hash = latest_committed_eb; + globals.latest_synced_eb = block_data.to_simple(); + globals.latest_prepared_eb_hash = latest_committed_eb; + globals.latest_finalized_mb_hash = latest_committed_mb; + globals.latest_computed_mb_hash = latest_committed_mb; + }); + + // Malachite is still a `MalachiteServiceStarter` at this point (its core + // only launches in `run_inner`), so no consensus callbacks can race this + // restoration. Once started, Malachite value-syncs from genesis and replays + // the chain; the service's startup gate suppresses every replayed event up + // to and including `latest_committed_mb`, then resumes live consensus. + + log::info!( + "Fast synchronization done: synced to {latest_committed_eb:?}, height {height:?}, MB {latest_committed_mb}", + height = block_data.header.height ); + + #[cfg(test)] + sender + .send(crate::tests::utils::TestingEvent::FastSyncDone( + replay_target, + )) + .await; + + Ok(Some(replay_target)) +} + +#[cfg(test)] +mod tests { + use super::*; + use ethexe_common::{BlockHeader, Digest, SimpleBlockData}; + use ethexe_ethereum::IntoBlockId; + use ethexe_processor::Processor; + use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + ops::RangeInclusive, + }; + + #[derive(Default)] + struct TestBlockLoader { + blocks: BTreeMap, + } + + impl BlockLoader for TestBlockLoader { + async fn load_simple(&self, _block: impl IntoBlockId) -> Result { + anyhow::bail!("load_simple is not used by EventData::collect tests") + } + + async fn load(&self, hash: H256, _header: Option) -> Result { + self.blocks + .get(&hash) + .cloned() + .with_context(|| format!("missing test block {hash}")) + } + + async fn load_many(&self, _range: RangeInclusive) -> Result> { + anyhow::bail!("load_many is not used by EventData::collect tests") + } + } + + #[tokio::test] + async fn instrument_codes_skips_already_processed_codes() { + let db = Database::memory(); + let code_id = CodeId::from([1; 32]); + db.set_code_valid(code_id, true); + let processor = Processor::new(db.clone()).unwrap(); + let mut compute = ComputeService::new(db.clone(), processor); + + instrument_codes(&mut compute, &db, BTreeSet::from([code_id])) + .await + .unwrap(); + } + + fn test_block(hash: H256, parent_hash: H256, events: Vec) -> BlockData { + BlockData { + hash, + header: BlockHeader { + height: hash.to_low_u64_be() as u32, + timestamp: 0, + parent_hash, + }, + events, + } + } + + #[tokio::test] + async fn event_data_collects_committed_chain_as_single_pair() { + let db = Database::memory(); + let older_block = H256::from_low_u64_be(1); + let newer_block = H256::from_low_u64_be(2); + let older_mb = H256::repeat_byte(0x11); + let older_eb = H256::repeat_byte(0x22); + let newer_mb = H256::repeat_byte(0x33); + let newer_eb = H256::repeat_byte(0x44); + let newer_batch = Digest::random(); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + newer_block, + test_block( + newer_block, + older_block, + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(newer_mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(newer_eb))), + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest: newer_batch, + })), + ], + ), + ); + loader.blocks.insert( + older_block, + test_block( + older_block, + H256::zero(), + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(older_mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(older_eb))), + ], + ), + ); + + let event_data = EventData::collect(&loader, &db, newer_block) + .await + .unwrap() + .expect("committed chain found"); + + assert_eq!(event_data.replay_target.mb_hash, newer_mb); + assert_eq!(event_data.replay_target.eb_hash, newer_eb); + assert_eq!(event_data.latest_committed_batch, newer_batch); + } + + #[tokio::test] + async fn event_data_skips_newer_mb_without_eb_anchor() { + let db = Database::memory(); + let older_block = H256::from_low_u64_be(1); + let newer_block = H256::from_low_u64_be(2); + let older_mb = H256::repeat_byte(0x11); + let older_eb = H256::repeat_byte(0x22); + let newer_mb = H256::repeat_byte(0x33); + let older_batch = Digest::random(); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + newer_block, + test_block( + newer_block, + older_block, + vec![BlockEvent::Router(RouterEvent::MBCommitted( + MBCommittedEvent(newer_mb), + ))], + ), + ); + loader.blocks.insert( + older_block, + test_block( + older_block, + H256::zero(), + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(older_mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(older_eb))), + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest: older_batch, + })), + ], + ), + ); + + let event_data = EventData::collect(&loader, &db, newer_block) + .await + .unwrap() + .expect("committed chain found"); + + assert_eq!(event_data.replay_target.mb_hash, older_mb); + assert_eq!(event_data.replay_target.eb_hash, older_eb); + assert_eq!(event_data.latest_committed_batch, older_batch); + } + + #[tokio::test] + async fn event_data_binds_batch_digest_to_selected_replay_target() { + let db = Database::memory(); + let older_block = H256::from_low_u64_be(1); + let newer_block = H256::from_low_u64_be(2); + let older_mb = H256::repeat_byte(0x11); + let older_eb = H256::repeat_byte(0x22); + let older_batch = Digest::random(); + let newer_batch = Digest::random(); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + newer_block, + test_block( + newer_block, + older_block, + vec![BlockEvent::Router(RouterEvent::BatchCommitted( + BatchCommittedEvent { + digest: newer_batch, + }, + ))], + ), + ); + loader.blocks.insert( + older_block, + test_block( + older_block, + H256::zero(), + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(older_mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(older_eb))), + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest: older_batch, + })), + ], + ), + ); + + let event_data = EventData::collect(&loader, &db, newer_block) + .await + .unwrap() + .expect("committed chain found"); + + assert_eq!(event_data.replay_target.mb_hash, older_mb); + assert_eq!(event_data.replay_target.eb_hash, older_eb); + assert_eq!(event_data.latest_committed_batch, older_batch); + } + + #[tokio::test] + async fn event_data_keeps_latest_batch_digest_from_selected_block() { + let db = Database::memory(); + let block = H256::from_low_u64_be(1); + let mb = H256::repeat_byte(0x11); + let eb = H256::repeat_byte(0x22); + let chain_batch = Digest::random(); + let later_batch = Digest::random(); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + block, + test_block( + block, + H256::zero(), + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(eb))), + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest: chain_batch, + })), + BlockEvent::Router(RouterEvent::BatchCommitted(BatchCommittedEvent { + digest: later_batch, + })), + ], + ), + ); + + let event_data = EventData::collect(&loader, &db, block) + .await + .unwrap() + .expect("committed chain found"); + + assert_eq!(event_data.replay_target.mb_hash, mb); + assert_eq!(event_data.replay_target.eb_hash, eb); + assert_eq!(event_data.latest_committed_batch, later_batch); + } + + #[tokio::test] + async fn event_data_rejects_committed_chain_without_batch_digest() { + let db = Database::memory(); + let block = H256::from_low_u64_be(1); + let mb = H256::repeat_byte(0x11); + let eb = H256::repeat_byte(0x22); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + block, + test_block( + block, + H256::zero(), + vec![ + BlockEvent::Router(RouterEvent::MBCommitted(MBCommittedEvent(mb))), + BlockEvent::Router(RouterEvent::EBCommitted(EBCommittedEvent(eb))), + ], + ), + ); + + let Err(err) = EventData::collect(&loader, &db, block).await else { + panic!("expected missing batch digest error"); + }; + + assert!( + err.to_string() + .contains("committed MB replay target without BatchCommitted event") + ); + } + + #[tokio::test] + async fn event_data_ignores_committed_mb_without_eb_anchor() { + let db = Database::memory(); + let block = H256::from_low_u64_be(1); + let mb = H256::repeat_byte(0x11); + + let mut loader = TestBlockLoader::default(); + loader.blocks.insert( + block, + test_block( + block, + H256::zero(), + vec![BlockEvent::Router(RouterEvent::MBCommitted( + MBCommittedEvent(mb), + ))], + ), + ); + + assert!( + EventData::collect(&loader, &db, block) + .await + .unwrap() + .is_none() + ); + } } diff --git a/ethexe/service/src/lib.rs b/ethexe/service/src/lib.rs index b4b2a23c2ca..1d8824acf9f 100644 --- a/ethexe/service/src/lib.rs +++ b/ethexe/service/src/lib.rs @@ -36,17 +36,16 @@ use crate::{ pending_tx::PendingNetworkInjectedTx, }; use alloy::{ + eips::BlockId, node_bindings::{Anvil, AnvilInstance}, providers::{ProviderBuilder, RootProvider, ext::AnvilApi}, rpc::types::anvil::Metadata, }; use anyhow::{Context, Result, bail}; -use async_trait::async_trait; use ethexe_blob_loader::{BlobLoader, BlobLoaderEvent, BlobLoaderService, ConsensusLayerConfig}; use ethexe_common::{ CodeAndIdUnchecked, PromiseEmissionMode, db::{GlobalsStorageRW, MbStorageRO}, - gear::CodeState, injected::{CompactPromise, InjectedTransactionAcceptance, Receipt}, network::VerifiedValidatorMessage, }; @@ -60,23 +59,17 @@ use ethexe_malachite::{ InjectedTxMempool, MalachiteEvent, MalachiteServiceConfig, MalachiteServiceStarter, ValidatorEntry, }; -use ethexe_network::{ - NetworkEvent, NetworkRuntimeConfig, NetworkService, TransportType, - db_sync::ExternalDataProvider, -}; -use ethexe_observer::{ - ObserverConfig, ObserverEvent, ObserverService, - utils::{BlockId, BlockLoader}, -}; +use ethexe_network::{NetworkEvent, NetworkRuntimeConfig, NetworkService, TransportType}; +use ethexe_observer::{ObserverConfig, ObserverEvent, ObserverService, utils::BlockLoader}; use ethexe_processor::{ProcessedCodeInfo, Processor, ProcessorConfig, ValidCodeInfo}; use ethexe_prometheus::{PrometheusEvent, PrometheusService}; use ethexe_rpc::{RpcEvent, RpcServer}; use ethexe_service_utils::{OptionFuture as _, OptionStreamNext as _}; use futures::{FutureExt, StreamExt}; -use gprimitives::{ActorId, CodeId, H256}; +use gprimitives::CodeId; use gsigner::secp256k1::{Address, PrivateKey, PublicKey, Secp256k1SignerExt, Signer}; use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, + collections::{BTreeMap, HashMap}, num::NonZero, path::PathBuf, pin::Pin, @@ -103,32 +96,6 @@ pub enum Event { Prometheus(PrometheusEvent), } -#[derive(Clone)] -struct RouterDataProvider(RouterQuery); - -#[async_trait] -impl ExternalDataProvider for RouterDataProvider { - fn clone_boxed(&self) -> Box { - Box::new(self.clone()) - } - - async fn programs_code_ids_at( - self: Box, - program_ids: BTreeSet, - block: H256, - ) -> Result> { - self.0.programs_code_ids_at(program_ids, block).await - } - - async fn codes_states_at( - self: Box, - code_ids: BTreeSet, - block: H256, - ) -> Result> { - self.0.codes_states_at(code_ids, block).await - } -} - /// Build the Malachite validator set from the on-chain validator /// list (in router order) by looking each address up in the /// `address -> public key` table loaded from the @@ -369,7 +336,7 @@ impl Service { let initial_chain_head = observer .block_loader() - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .context("failed to get latest block")?; @@ -472,7 +439,6 @@ impl Service { validator_key: validator_pub_key, general_signer: signer.clone(), network_signer, - external_data_provider: Box::new(RouterDataProvider(router_query)), db: db.clone(), }; @@ -614,16 +580,18 @@ impl Service { } pub async fn run(mut self) -> Result<()> { - if self.fast_sync { - fast_sync::sync(&mut self).await?; - } + let replay_target = if self.fast_sync { + fast_sync::sync(&mut self).await? + } else { + None + }; - self.run_inner().await.inspect_err(|err| { + self.run_inner(replay_target).await.inspect_err(|err| { log::error!("Service finished work with error: {err:?}"); }) } - async fn run_inner(self) -> Result<()> { + async fn run_inner(self, replay_target: Option) -> Result<()> { let Service { db, mut network, @@ -637,13 +605,11 @@ impl Service { rpc, fast_sync: _, validator_pub_key, - shutdown_rx, + mut shutdown_rx, #[cfg(test)] sender, } = self; - let mut shutdown_rx = shutdown_rx; - let mut malachite = malachite_starter.start().await?; if let Some(validator_proof) = malachite.validator_proof() { @@ -673,6 +639,10 @@ impl Service { let mut network_injected_txs: HashMap<_, PendingNetworkInjectedTx> = HashMap::new(); + // Suppresses Malachite's startup replay of the chain fast sync already + // restored, up to and including the committed target MB. + let mut replay_gate = fast_sync::ReplayGate::new(replay_target); + loop { let event: Event = tokio::select! { event = compute.select_next_some() => event?.into(), @@ -735,8 +705,9 @@ impl Service { if let Some(c) = consensus.as_mut() { c.receive_prepared_block(block_hash)?; } - - malachite.receive_eb_prepared(block_hash).await; + // Compute owns the "wait for the prerequisite EB" gate + // now (see `ComputeSubService`), so Malachite no longer + // needs a prepared-block notification. } ComputeEvent::CodeProcessed(_) => { // Nothing @@ -946,6 +917,13 @@ impl Service { "Malachite: BlockProposal", ); + // Suppress replay of the chain fast sync already + // restored, up to and including the committed target MB. + if !replay_gate.allow(&db, mb_hash)? { + tracing::debug!(mb_hash = %mb_hash, "fast-sync replay gate: skipping replayed proposal"); + continue; + } + compute.compute_mb(mb_hash, ethexe_common::PromisePolicy::Enabled); } MalachiteEvent::BlockFinalized { diff --git a/ethexe/service/src/tests/mod.rs b/ethexe/service/src/tests/mod.rs index 1bf3f9789a5..f7869186419 100644 --- a/ethexe/service/src/tests/mod.rs +++ b/ethexe/service/src/tests/mod.rs @@ -5,17 +5,23 @@ pub(crate) mod utils; -use crate::tests::utils::{ - EnvNetworkConfig, GenesisInitializerFromDump, InfiniteStreamExt, NodeConfig, TestEnv, - TestEnvConfig, TestingEvent, TestingNetworkEvent, TestingRpcEvent, ValidatorsConfig, - init_logger, stop_nodes, test_info, +use crate::{ + fast_sync::FastSyncReplayTarget, + tests::utils::{ + EnvNetworkConfig, GenesisInitializerFromDump, InfiniteStreamExt, Node, NodeConfig, TestEnv, + TestEnvConfig, TestingEvent, TestingNetworkEvent, TestingRpcEvent, ValidatorsConfig, + init_logger, stop_nodes, test_info, + }, }; use alloy::{ primitives::U256, providers::{Provider as _, WalletProvider, ext::AnvilApi}, }; use ethexe_common::{ - db::{CodesStorageRO, GlobalsStorageRO, InjectedStorageRO, MbStorageRO, OnChainStorageRO}, + db::{ + BlockMetaStorageRO, CodesStorageRO, GlobalsStorageRO, InjectedStorageRO, MbStorageRO, + OnChainStorageRO, + }, ecdsa::ContractSignature, events::{ BlockEvent, MirrorEvent, @@ -28,7 +34,7 @@ use ethexe_common::{ mock::*, }; use ethexe_consensus::BatchCommitter; -use ethexe_db::{Database, dump::StateDump}; +use ethexe_db::{Database, dump::StateDump, verifier::IntegrityVerifier}; use ethexe_ethereum::{EthereumBuilder, TryGetReceipt, router::Router}; use ethexe_processor::Processor; use ethexe_rpc::{InjectedClient, ProgramClient}; @@ -1476,9 +1482,7 @@ async fn reorg_deeper_than_quarantine() { let latest_block = env.latest_block().await; test_info!("Waiting for {latest_block} to be finalized in MB"); - node.events() - .wait_till_eth_block_finalized_in_mb(latest_block.hash) - .await; + node.events().find_mb_advanced_eb(latest_block.hash).await; test_info!("📗 Reverting Anvil to deep snapshot — past quarantine"); env.provider @@ -1496,7 +1500,7 @@ async fn reorg_deeper_than_quarantine() { let mut receiver = node.new_events(); // Here we take in account kicking stream - latest_block is not passed quarantine yet, // but kicks will generate new anvil blocks, but still this block cannot be finalized in mb, because branch is broken. - let waiting_future = receiver.wait_till_eth_block_finalized_in_mb(latest_block.hash); + let waiting_future = receiver.find_mb_advanced_eb(latest_block.hash); tokio::time::timeout(Duration::from_secs(20), waiting_future) .await .expect_err("block should not be finalized within 20 seconds after deep reorg"); @@ -3307,8 +3311,203 @@ async fn reply_callback() { } #[tokio::test] -#[ignore = "TODO: #5487 port to MB-driven test harness"] -async fn fast_sync() {} +#[ntest::timeout(120_000)] +async fn fast_sync() { + init_logger(); + + let assert_chain = |latest_block, replay_target, alice: &Node, bob: &Node| { + let FastSyncReplayTarget { + eb_hash: fast_synced_eb, + mb_hash: fast_synced_mb, + } = replay_target; + + log::info!("Assert chain in range {latest_block}..{fast_synced_eb}"); + + IntegrityVerifier::new(alice.db.clone()) + .verify_chain(latest_block, fast_synced_eb) + .expect("failed to verify Alice database"); + + IntegrityVerifier::new(bob.db.clone()) + .verify_chain(latest_block, fast_synced_eb) + .expect("failed to verify Bob database"); + + let mut eb_hash = latest_block; + loop { + if fast_synced_eb == eb_hash { + break; + } + + log::trace!("assert EB {eb_hash}"); + + let alice_meta = alice.db.block_meta(eb_hash); + let bob_meta = bob.db.block_meta(eb_hash); + // we do not compare all fields of meta because they can vary and it's ok + assert_eq!(alice_meta.prepared, bob_meta.prepared); + assert_eq!( + alice_meta.latest_era_validators_committed, + bob_meta.latest_era_validators_committed + ); + + assert_eq!(alice.db.block_header(eb_hash), bob.db.block_header(eb_hash)); + assert_eq!(alice.db.block_events(eb_hash), bob.db.block_events(eb_hash)); + assert_eq!(alice.db.block_synced(eb_hash), bob.db.block_synced(eb_hash)); + + let header = alice.db.block_header(eb_hash).unwrap(); + eb_hash = header.parent_hash; + + let mb_hash = bob_meta.last_committed_mb.unwrap(); + if mb_hash == fast_synced_mb { + continue; + } + + let alice_compact_block = alice.db.mb_compact_block(mb_hash).unwrap(); + let bob_compact_block = bob.db.mb_compact_block(mb_hash).unwrap(); + assert_eq!(alice_compact_block, bob_compact_block); + + let alice_operations = alice.db.operations(alice_compact_block.operations_hash); + let bob_operations = bob.db.operations(bob_compact_block.operations_hash); + assert_eq!(alice_operations, bob_operations); + + let alice_program_states = alice.db.mb_program_states(mb_hash); + let bob_program_states = bob.db.mb_program_states(mb_hash); + assert_eq!(alice_program_states, bob_program_states); + + let alice_outcome = alice.db.mb_outcome(mb_hash); + let bob_outcome = bob.db.mb_outcome(mb_hash); + assert_eq!(alice_outcome, bob_outcome); + + let alice_schedule = alice.db.mb_schedule(mb_hash); + let bob_schedule = bob.db.mb_schedule(mb_hash); + assert_eq!(alice_schedule, bob_schedule); + + let alice_meta = alice.db.mb_meta(mb_hash); + let bob_meta = bob.db.mb_meta(mb_hash); + assert_eq!(alice_meta, bob_meta); + } + }; + + let config = TestEnvConfig { + network: EnvNetworkConfig::Enabled, + // Keep proposals one EB behind the freshest head. On loaded CI, + // Bob can receive Alice's proposal before its Malachite chain-head + // register observes the just-mined EB. + post_quarantine_delay: 1, + kicking_per_blocks: Some(1), + ..Default::default() + }; + let mut env = TestEnv::new(config).await.unwrap(); + + test_info!("📗 Starting Alice"); + let mut alice = env + .new_node(NodeConfig::named("Alice").validator(env.validators[0])) + .await; + alice.start_service().await; + + test_info!("📗 Creating `demo-mul-by-const` programs"); + + let code_info = env + .upload_code(demo_mul_by_const::WASM_BINARY) + .await + .unwrap() + .wait_for() + .await + .unwrap(); + + let code_id = code_info.code_id; + let mut program_ids = [ActorId::zero(); 8]; + + for (i, program_id) in program_ids.iter_mut().enumerate() { + let program_info = env + .create_program_with_params(code_id, H256([i as u8; 32]), None, 500_000_000_000_000) + .await + .unwrap() + .wait_for() + .await + .unwrap(); + + *program_id = program_info.program_id; + + let value = i as u64 % 3; + let _reply_info = env + .send_message(program_info.program_id, &value.encode()) + .await + .unwrap() + .wait_for() + .await + .unwrap(); + } + + let latest_block = env.latest_block().await.hash; + alice.events().find_mb_advanced_eb(latest_block).await; + + test_info!("Starting Bob (fast-sync)"); + let mut bob = env.new_node(NodeConfig::named("Bob").fast_sync()).await; + bob.start_service().await; + + test_info!("📗 Sending messages to programs"); + + for (i, program_id) in program_ids.into_iter().enumerate() { + let reply_info = env + .send_message(program_id, &(i as u64).encode()) + .await + .unwrap() + .wait_for() + .await + .unwrap(); + assert_eq!( + reply_info.code, + ReplyCode::Success(SuccessReplyReason::Manual) + ); + } + + let latest_block = env.latest_block().await.hash; + let mut alice_events = alice.events(); + let mut bob_events = bob.events(); + tokio::join!( + alice_events.find_mb_computed_eb(latest_block), + bob_events.find_mb_computed_eb(latest_block) + ); + + test_info!("📗 Stopping Bob"); + bob.stop_service().await; + + let replay_target = bob.latest_fast_synced_blocks.unwrap(); + assert_chain(latest_block, replay_target, &alice, &bob); + + for (i, program_id) in program_ids.into_iter().enumerate() { + let i = (i * 3) as u64; + let reply_info = env + .send_message(program_id, &i.encode()) + .await + .unwrap() + .wait_for() + .await + .unwrap(); + assert_eq!( + reply_info.code, + ReplyCode::Success(SuccessReplyReason::Manual) + ); + } + + env.skip_blocks(100).await; + + let latest_block = env.latest_block().await.hash; + alice.events().find_mb_advanced_eb(latest_block).await; + + test_info!("📗 Starting Bob again to check how it handles partially empty database"); + bob.start_service().await; + + let latest_block = env.latest_block().await.hash; + let mut alice_events = alice.events(); + let mut bob_events = bob.events(); + tokio::join!( + alice_events.find_mb_computed_eb(latest_block), + bob_events.find_mb_computed_eb(latest_block) + ); + + let replay_target = bob.latest_fast_synced_blocks.unwrap(); + assert_chain(latest_block, replay_target, &alice, &bob); +} #[tokio::test] #[ntest::timeout(120_000)] @@ -3356,9 +3555,7 @@ async fn re_genesis_with_state_dump() { // Ensure the ping-pong state is committed on-chain before re-genesis, so the // dump taken at the new genesis block carries it. let latest_block = env.latest_block().await.hash; - node.events() - .wait_till_eth_block_finalized_in_mb(latest_block) - .await; + node.events().find_mb_advanced_eb(latest_block).await; log::info!( "📗 Phase 2: re-genesis the router via reinitialize + lookupGenesisHash. \ @@ -3380,9 +3577,7 @@ async fn re_genesis_with_state_dump() { // Wait until the node commits an MB covering the new genesis block before // dumping from its DB. - node.events() - .wait_till_eth_block_finalized_in_mb(new_genesis_hash) - .await; + node.events().find_mb_advanced_eb(new_genesis_hash).await; log::info!("📗 Phase 3: collect state dump at the new genesis block."); let dump = StateDump::collect_from_storage(&node.db, new_genesis_hash).unwrap(); @@ -3530,9 +3725,7 @@ async fn re_genesis_delayed_message() { // Ensure the delayed-send state is committed before re-genesis. let latest_block = env.latest_block().await.hash; - node.events() - .wait_till_eth_block_finalized_in_mb(latest_block) - .await; + node.events().find_mb_advanced_eb(latest_block).await; // Phase 2: re-genesis via reinitialize + lookupGenesisHash; the new genesis // is the block where the reinitialize tx was mined. @@ -3553,9 +3746,7 @@ async fn re_genesis_delayed_message() { // Wait until the node commits an MB covering the new genesis block before // dumping from its DB. - node.events() - .wait_till_eth_block_finalized_in_mb(new_genesis_hash) - .await; + node.events().find_mb_advanced_eb(new_genesis_hash).await; // Phase 3: collect dump at the new genesis block; it should still carry the // pending delayed send in the dispatch stash because the 5-block delay diff --git a/ethexe/service/src/tests/utils/env.rs b/ethexe/service/src/tests/utils/env.rs index 7c4250b5a2a..63b8e9cd888 100644 --- a/ethexe/service/src/tests/utils/env.rs +++ b/ethexe/service/src/tests/utils/env.rs @@ -2,14 +2,16 @@ // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 use crate::{ - RouterDataProvider, Service, + Service, config::EthereumConfig, + fast_sync::FastSyncReplayTarget, tests::utils::{ InfiniteStreamExt, TestingEvent, TestingNetworkEvent, events::{self, ObserverEventReceiver, ObserverEventSender, TestingEventReceiver}, }, }; use alloy::{ + eips::BlockId, node_bindings::{Anvil, AnvilInstance}, providers::{ProviderBuilder, RootProvider, ext::AnvilApi}, rpc::types::anvil::{Metadata, MineOptions}, @@ -38,12 +40,13 @@ use ethexe_ethereum::{ router::RouterQuery, }; use ethexe_malachite::{ - InjectedTxMempool, MalachiteServiceConfig, MalachiteServiceStarter, ValidatorEntry, + InjectedTxMempool, MalachiteConfigEnvironment, MalachiteServiceConfig, MalachiteServiceStarter, + ValidatorEntry, }; use ethexe_network::{NetworkConfig, NetworkRuntimeConfig, NetworkService, export::Multiaddr}; use ethexe_observer::{ ObserverConfig, ObserverService, - utils::{BlockId, BlockLoader, EthereumBlockLoader}, + utils::{BlockLoader, EthereumBlockLoader}, }; use ethexe_processor::{DEFAULT_CHUNK_SIZE, Processor}; use ethexe_rpc::{DEFAULT_BLOCK_GAS_LIMIT_MULTIPLIER, RpcConfig, RpcServer}; @@ -257,7 +260,7 @@ impl TestEnv { .unwrap(); let latest_block = observer .block_loader() - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .context("failed to get latest block")?; let latest_validators = router_query @@ -320,7 +323,6 @@ impl TestEnv { validator_key: None, general_signer: signer.clone(), network_signer: signer.clone(), - external_data_provider: Box::new(RouterDataProvider(router_query.clone())), db: db.clone(), }; @@ -432,7 +434,7 @@ impl TestEnv { name, db, multiaddr: None, - latest_fast_synced_block: None, + latest_fast_synced_blocks: None, custom_committer: None, router_query: self.router_query.clone(), eth_cfg: self.eth_cfg.clone(), @@ -699,7 +701,7 @@ impl TestEnv { pub async fn latest_block(&self) -> SimpleBlockData { EthereumBlockLoader::new(self.provider.clone(), self.eth_cfg.router_address) - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .unwrap() } @@ -916,7 +918,7 @@ pub struct Node { pub name: Option, pub db: Database, pub multiaddr: Option, - pub latest_fast_synced_block: Option, + pub latest_fast_synced_blocks: Option, pub custom_committer: Option>, router_query: RouterQuery, @@ -970,7 +972,7 @@ impl Node { .unwrap(); let latest_block = observer .block_loader() - .load_simple(BlockId::Latest) + .load_simple(BlockId::latest()) .await .unwrap(); let latest_validators = observer @@ -1064,6 +1066,7 @@ impl Node { let mut mc = MalachiteServiceConfig::from_home_dir(home_path).with_validators(validators); + mc.env = MalachiteConfigEnvironment::Test; mc.canonical_quarantine = self.canonical_quarantine; mc.post_quarantine_delay = self.post_quarantine_delay; // Tests drive content in bursts; a short propose timeout keeps @@ -1141,9 +1144,12 @@ impl Node { self.shutdown_tx = Some(shutdown_tx); if self.fast_sync { - self.latest_fast_synced_block = Some( + self.latest_fast_synced_blocks = Some( self.events() - .find_map(|event| event.try_unwrap_fast_sync_done().ok()) + .find_map(|event| match event { + TestingEvent::FastSyncDone(blocks) => Some(blocks), + _ => None, + }) .await, ); } @@ -1238,7 +1244,6 @@ impl Node { validator_key: self.validator_config.as_ref().map(|c| c.public_key), general_signer: self.signer.clone(), network_signer: self.signer.clone(), - external_data_provider: Box::new(RouterDataProvider(self.router_query.clone())), db: self.db.clone(), }; @@ -1260,7 +1265,7 @@ impl Node { let provider = RootProvider::connect(&self.eth_cfg.rpc).await.unwrap(); let block_loader = EthereumBlockLoader::new(provider, self.eth_cfg.router_address); - let latest_block = block_loader.load_simple(BlockId::Latest).await.unwrap(); + let latest_block = block_loader.load_simple(BlockId::latest()).await.unwrap(); let latest_validators = self .router_query .validators_at(latest_block.hash) diff --git a/ethexe/service/src/tests/utils/events.rs b/ethexe/service/src/tests/utils/events.rs index 4c8530ecd5e..b83a16c63a0 100644 --- a/ethexe/service/src/tests/utils/events.rs +++ b/ethexe/service/src/tests/utils/events.rs @@ -3,7 +3,7 @@ #![allow(clippy::double_parens)] // produced by `derive_more::TryUnwrap` -use crate::Event; +use crate::{Event, fast_sync::FastSyncReplayTarget}; use alloy::providers::{RootProvider, ext::AnvilApi}; use async_broadcast::{Receiver, RecvError, Sender}; use ethexe_blob_loader::BlobLoaderEvent; @@ -121,11 +121,10 @@ impl TestingRpcEvent { } } -#[derive(Debug, Clone, Eq, PartialEq, derive_more::TryUnwrap)] +#[derive(Debug, Clone, Eq, PartialEq)] pub enum TestingEvent { // Fast sync done. Sent just once. - #[allow(dead_code)] - FastSyncDone(H256), + FastSyncDone(FastSyncReplayTarget), // Basic event to notify that service has started. Sent just once. ServiceStarted, // Services events. @@ -309,7 +308,6 @@ impl KickingStream> { } impl TestingEventReceiver { - #[allow(dead_code)] pub async fn find_block_synced(&mut self) -> H256 { self.find_map(|event| { if let TestingEvent::Observer(ObserverEvent::BlockSynced(block_hash)) = event { @@ -323,7 +321,6 @@ impl TestingEventReceiver { /// Drive the compute stream forward until a `BlockPrepared(target)` event /// arrives. - #[allow(dead_code)] pub async fn find_block_prepared(&mut self, target: H256) -> H256 { self.find_map(|event| match event { TestingEvent::Compute(ComputeEvent::BlockPrepared(h)) if h == target => Some(h), @@ -333,10 +330,13 @@ impl TestingEventReceiver { } /// Wait until any MB becomes computed, returning its hash. - #[allow(dead_code)] - pub async fn find_any_mb_computed(&mut self) -> H256 { + pub async fn find_mb_computed(&mut self, mb_hash: H256) -> H256 { self.find_map(|event| match event { - TestingEvent::Compute(ComputeEvent::MbComputed(mb_hash)) => Some(mb_hash), + TestingEvent::Compute(ComputeEvent::MbComputed(event_mb_hash)) + if event_mb_hash == mb_hash => + { + Some(mb_hash) + } _ => None, }) .await @@ -347,8 +347,7 @@ impl TestingEventReceiver { /// `AdvanceTillEthereumBlock` transaction — it suffices that it is an /// ancestor of this MB's `last_advanced_eb` (i.e., it sits inside /// the eth-chain segment this MB advanced over). - #[allow(dead_code)] - pub async fn wait_till_eth_block_finalized_in_mb(&mut self, target_eth_block: H256) { + pub async fn find_mb_advanced_eb(&mut self, eb_hash: H256) -> H256 { self.find_map_with_db(|db, event| { let TestingEvent::Malachite(MalachiteEvent::BlockFinalized { mb_hash, .. }) = event else { @@ -368,8 +367,8 @@ impl TestingEventReceiver { // covers it. let mut cursor = last_advanced; while cursor != prev_advanced { - if cursor == target_eth_block { - return Some(()); + if cursor == eb_hash { + return Some(mb_hash); } let header = db.block_header(cursor)?; if header.parent_hash.is_zero() { @@ -382,6 +381,25 @@ impl TestingEventReceiver { .await } + /// Wait until an MB covering `eb_hash` is finalized and computed, + /// returning that MB hash. + /// + /// The finalized event identifies which MB advanced over the requested EB; + /// computation may already have happened by then, so this also accepts the + /// persisted `computed` marker instead of requiring a later event. + pub async fn find_mb_computed_eb(&mut self, eb_hash: H256) -> H256 { + let mb_hash = self.find_mb_advanced_eb(eb_hash).await; + + // `MbComputed` can be emitted before the matching finalization event + // that lets us map `eb_hash` to `mb_hash`; don't wait for a duplicate + // event after compute already persisted its marker. + if self.db().mb_meta(mb_hash).computed { + return mb_hash; + } + + self.find_mb_computed(mb_hash).await + } + pub async fn find_map_with_db( &mut self, mut f: impl FnMut(Database, TestingEvent) -> Option, @@ -443,3 +461,63 @@ impl ObserverEventReceiver { KickingStream::new(stream, kicks) } } + +#[cfg(test)] +mod tests { + use super::*; + use ethexe_common::{ + BlockHeader, + db::{CompactMb, OnChainStorageRW}, + }; + use ethexe_db::Database; + use ethexe_malachite::CommitCertificate; + + #[tokio::test] + async fn find_mb_computed_eb_returns_when_mb_was_already_computed() { + let db = Database::memory(); + let eb_hash = H256::from_low_u64_be(1); + let mb_hash = H256::from_low_u64_be(2); + + db.set_block_header( + eb_hash, + BlockHeader { + height: 1, + timestamp: 12, + parent_hash: H256::zero(), + }, + ); + db.set_mb_compact_block( + mb_hash, + CompactMb { + parent: H256::zero(), + height: 1, + operations_hash: H256::zero(), + }, + ); + db.mutate_mb_meta(mb_hash, |meta| { + meta.computed = true; + meta.last_advanced_eb = eb_hash; + }); + + let (sender, mut receiver) = channel(db, None); + sender + .send(TestingEvent::Malachite(MalachiteEvent::BlockFinalized { + cert: CommitCertificate { + height: 1, + mb_hash, + signatures: Vec::new(), + }, + height: 1, + mb_hash, + })) + .await; + + let found = receiver.find_mb_computed_eb(eb_hash); + futures::pin_mut!(found); + let found = futures::future::poll_immediate(&mut found) + .await + .expect("already-computed MB should not wait for another MbComputed event"); + + assert_eq!(found, mb_hash); + } +} diff --git a/ethexe/service/src/tests/utils/mod.rs b/ethexe/service/src/tests/utils/mod.rs index 0b164821cc6..02d07d9de98 100644 --- a/ethexe/service/src/tests/utils/mod.rs +++ b/ethexe/service/src/tests/utils/mod.rs @@ -2,14 +2,14 @@ // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 pub use env::*; -use ethexe_db::{GenesisInitializer, dump::StateDump}; -use ethexe_processor::Processor; pub use events::*; -use futures::FutureExt; mod env; mod events; +use ethexe_db::{GenesisInitializer, dump::StateDump}; +use ethexe_processor::Processor; +use futures::FutureExt; use tracing_subscriber::EnvFilter; pub fn init_logger() { diff --git a/test_doc_attr b/test_doc_attr new file mode 100755 index 00000000000..7cf9079975f Binary files /dev/null and b/test_doc_attr differ diff --git a/utils/gear-workspace-hack/Cargo.toml b/utils/gear-workspace-hack/Cargo.toml index 122d48cb376..4b6ab913953 100644 --- a/utils/gear-workspace-hack/Cargo.toml +++ b/utils/gear-workspace-hack/Cargo.toml @@ -210,6 +210,7 @@ features = ["rocksdb", "test-helpers"] ### BEGIN HAKARI SECTION [target.'cfg(not(any(target_arch = "wasm32", loom)))'.dependencies] +aead = { version = "0.5", default-features = false, features = ["getrandom", "std"] } ahash = { version = "0.8" } alloy = { version = "2", features = ["kzg", "node-bindings", "provider-anvil-api", "provider-ws", "rpc-types-beacon", "rpc-types-eth", "signer-mnemonic"] } alloy-chains = { version = "0.2" } @@ -250,10 +251,12 @@ byte-slice-cast = { version = "1", default-features = false, features = ["std"] bytemuck = { version = "1", default-features = false, features = ["const_zeroed", "derive", "min_const_generics", "must_cast"] } byteorder = { version = "1", features = ["i128"] } bytes = { version = "1", features = ["serde"] } +cipher = { version = "0.4", default-features = false, features = ["zeroize"] } clap = { version = "4", features = ["derive", "env", "string", "wrap_help"] } clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "string", "suggestions", "usage", "wrap_help"] } concurrent-queue = { version = "2" } const-hex = { version = "1", features = ["core-error", "serde"] } +core2 = { git = "https://github.com/bbqsrc/core2", rev = "545e84bcb0f235b12e21351e0c69767958efe2a7", default-features = false, features = ["std"] } cranelift-bitset = { version = "0.131", default-features = false, features = ["enable-serde"] } cranelift-codegen = { version = "0.131", default-features = false, features = ["host-arch", "pulley", "std", "timing", "unwind"] } crc32fast = { version = "1" } @@ -304,7 +307,7 @@ hash-db = { version = "0.16" } hash256-std-hasher = { version = "0.15", default-features = false, features = ["std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] } hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13", features = ["raw"] } -hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17", default-features = false, features = ["default-hasher"] } +hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17", default-features = false, features = ["allocator-api2", "default-hasher", "inline-more"] } hashbrown-986da7b5efc2b80e = { package = "hashbrown", version = "0.16", features = ["serde"] } hex = { version = "0.4", features = ["serde"] } hkdf = { version = "0.12", default-features = false, features = ["std"] } @@ -484,6 +487,7 @@ x25519-dalek = { version = "2", features = ["static_secrets"] } zeroize = { version = "1", features = ["derive", "std"] } [target.'cfg(not(any(target_arch = "wasm32", loom)))'.build-dependencies] +aead = { version = "0.5", default-features = false, features = ["getrandom", "std"] } ahash = { version = "0.8" } alloy = { version = "2", features = ["kzg", "node-bindings", "provider-anvil-api", "provider-ws", "rpc-types-beacon", "rpc-types-eth", "signer-mnemonic"] } alloy-chains = { version = "0.2" } @@ -528,10 +532,12 @@ bytemuck = { version = "1", default-features = false, features = ["const_zeroed" byteorder = { version = "1", features = ["i128"] } bytes = { version = "1", features = ["serde"] } cc = { version = "1", default-features = false, features = ["parallel"] } +cipher = { version = "0.4", default-features = false, features = ["zeroize"] } clap = { version = "4", features = ["derive", "env", "string", "wrap_help"] } clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "string", "suggestions", "usage", "wrap_help"] } concurrent-queue = { version = "2" } const-hex = { version = "1", features = ["core-error", "serde"] } +core2 = { git = "https://github.com/bbqsrc/core2", rev = "545e84bcb0f235b12e21351e0c69767958efe2a7", default-features = false, features = ["std"] } cranelift-bitset = { version = "0.131", default-features = false, features = ["enable-serde"] } cranelift-codegen = { version = "0.131", default-features = false, features = ["host-arch", "pulley", "std", "timing", "unwind"] } cranelift-codegen-meta = { version = "0.131", default-features = false, features = ["pulley"] } @@ -586,7 +592,7 @@ hash-db = { version = "0.16" } hash256-std-hasher = { version = "0.15", default-features = false, features = ["std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] } hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13", features = ["raw"] } -hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17", default-features = false, features = ["default-hasher"] } +hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17", default-features = false, features = ["allocator-api2", "default-hasher", "inline-more"] } hashbrown-986da7b5efc2b80e = { package = "hashbrown", version = "0.16", features = ["serde"] } hex = { version = "0.4", features = ["serde"] } hkdf = { version = "0.12", default-features = false, features = ["std"] }