From f48f8b3d8560520e5db0d0ae9dd09fa1e70d895e Mon Sep 17 00:00:00 2001 From: Muad'Dib Date: Tue, 18 Aug 2026 14:46:24 +0200 Subject: [PATCH 1/3] fix(sync): make unknown-parent requests an in-flight limit, not a lifetime budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three-slot flood guard on unknown-parent header requests was backed by a set that was only ever inserted into — never cleared when the parent arrived, never expired when none ever did. Three was therefore a budget for the life of the process rather than a cap on requests in flight. Once spent, the node could no longer ask for a missing parent at all, and recovered only if a peer volunteered the header unprompted. Observed 2026-08-18: a node that woke from suspend on an orphaned tip at 1,853,471 sat there 1h48m, receiving headers it could not attach, until a peer announced the one header it could no longer request. Slots are now released when the parent chains, and expire after 60s when it never arrives. Co-Authored-By: Claude Opus 5 (1M context) --- src/pipeline.rs | 122 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 113 insertions(+), 9 deletions(-) diff --git a/src/pipeline.rs b/src/pipeline.rs index 35fe493..3a39517 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -14,6 +14,48 @@ use tokio::sync::{mpsc, Mutex}; /// LRU buffer capacity for out-of-order headers (JVM: `headersCache` = 8192). const BUFFER_CAPACITY: usize = 8_192; +/// Maximum concurrent unknown-parent requests (flood guard). +const MAX_PARENT_REQUESTS: usize = 3; + +/// How long an unanswered parent request holds its slot. A parent no peer +/// will serve must not occupy the budget for the life of the process. +const PARENT_REQUEST_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +/// Unknown-parent header requests that are currently outstanding. +/// +/// Bounded so a batch of orphan headers cannot fan out into a request storm: +/// the fork chain links backward, so fetching the lowest missing parent is +/// enough to drain the rest of the buffer once it arrives. +#[derive(Default)] +struct ParentRequests { + in_flight: std::collections::HashMap<[u8; 32], std::time::Instant>, +} + +impl ParentRequests { + /// Decide whether to ask peers for `parent_id`, recording the request. + /// + /// Slots are released two ways — [`Self::resolved`] when the parent + /// arrives, and the TTL when no peer ever serves it. Release by neither + /// leaves the node unable to ask for a missing parent at all: it then + /// recovers only if some peer volunteers the header unprompted, which + /// is a matter of which peers it happens to be connected to. + fn should_request(&mut self, parent_id: [u8; 32], now: std::time::Instant) -> bool { + self.in_flight + .retain(|_, at| now.saturating_duration_since(*at) < PARENT_REQUEST_TTL); + if self.in_flight.contains_key(&parent_id) || self.in_flight.len() >= MAX_PARENT_REQUESTS { + return false; + } + self.in_flight.insert(parent_id, now); + true + } + + /// Record that a previously requested parent has arrived and chained, + /// releasing its slot. + fn resolved(&mut self, id: &[u8; 32]) { + self.in_flight.remove(id); + } +} + /// A modifier-store row: `(type_id, modifier_id, height, body_bytes, optional_aux_bytes)`. /// Used wherever we accumulate entries for `ModifierStore::put_batch`. type StoreEntry = (u8, [u8; 32], u32, Vec, Option>); @@ -39,8 +81,8 @@ pub struct ValidationPipeline { delivery_data_tx: mpsc::Sender, tracker: HeaderTracker, buffer: LruCache)>, - /// Parent IDs we've already requested for fork resolution (avoid flooding). - reorg_requested: std::collections::HashSet<[u8; 32]>, + /// Unknown-parent requests currently outstanding (avoid flooding). + reorg_requested: ParentRequests, /// Channel for forwarding unconfirmed transactions to the mempool task. tx_sender: Option)>>, } @@ -63,7 +105,7 @@ impl ValidationPipeline { delivery_data_tx, tracker: HeaderTracker::new(), buffer: LruCache::new(NonZeroUsize::new(BUFFER_CAPACITY).unwrap()), - reorg_requested: std::collections::HashSet::new(), + reorg_requested: ParentRequests::default(), tx_sender: None, } } @@ -301,6 +343,10 @@ impl ValidationPipeline { match chain.try_append(header.clone()) { Ok(AppendResult::Extended) => { chained += 1; + // If this header was an outstanding parent request, it has + // now arrived — release its slot rather than waiting out + // the TTL. + self.reorg_requested.resolved(&header_id.0 .0); let score_bytes = chain .score_at(header_height) .expect("score for just-appended header") @@ -323,6 +369,7 @@ impl ValidationPipeline { match chain.try_append(buf.clone()) { Ok(AppendResult::Extended) => { chained += 1; + self.reorg_requested.resolved(&bid.0 .0); let buf_score_bytes = chain .score_at(buf_height) .expect("score for just-appended buffered header") @@ -342,6 +389,7 @@ impl ValidationPipeline { } } Ok(AppendResult::Forked { fork_height }) => { + self.reorg_requested.resolved(&header_id.0 .0); // Header is valid but forks from the best chain. // Compute its cumulative score and store immediately // (later headers in this batch may extend this fork). @@ -457,12 +505,13 @@ impl ValidationPipeline { "ParentNotFound" ); - // Request the missing parent — but only the FIRST one per - // batch to avoid flooding. The fork chain links backward, - // so fetching the lowest missing parent is sufficient: once - // it arrives and chains, the rest drain from the buffer. - if (self.reorg_requested.is_empty() || self.reorg_requested.len() < 3) - && self.reorg_requested.insert(parent_id.0 .0) + // Request the missing parent, bounded by the in-flight + // budget. The fork chain links backward, so fetching the + // lowest missing parent is sufficient: once it arrives and + // chains, the rest drain from the buffer. + if self + .reorg_requested + .should_request(parent_id.0 .0, std::time::Instant::now()) { let _ = self .delivery_control_tx @@ -863,4 +912,59 @@ mod tests { let bytes2 = header2.scorex_serialize_bytes().unwrap(); assert_eq!(bytes, bytes2, "round-trip must be byte-identical"); } + + /// A parent request that has been answered must give its slot back. + /// + /// Regression: `reorg_requested` was only ever inserted into, so the + /// three-slot flood guard was a lifetime budget rather than an + /// in-flight limit. Once spent, a node could never again ask for a + /// missing parent — observed 2026-08-18, where a node that woke from + /// suspend on an orphaned tip sat at that height for 1h48m until a peer + /// happened to announce the header it could no longer request. + #[test] + fn resolved_parent_request_frees_its_slot() { + let (mut pipeline, _tx, _progress_rx, _ctrl_rx, _data_rx, _dir) = test_pipeline(); + let now = std::time::Instant::now(); + + for i in 1..=3u8 { + assert!( + pipeline.reorg_requested.should_request([i; 32], now), + "parent {i} must be requested while the in-flight budget has room" + ); + } + assert!( + !pipeline.reorg_requested.should_request([4; 32], now), + "a fourth concurrent request must be held back by the flood guard" + ); + + // The first parent arrives and chains — that request is done. + pipeline.reorg_requested.resolved(&[1; 32]); + + assert!( + pipeline.reorg_requested.should_request([4; 32], now), + "a resolved request must free its slot for the next unknown parent" + ); + } + + /// A parent request nobody answers must not hold its slot forever. + /// + /// The companion leak to [`resolved_parent_request_frees_its_slot`]: a + /// parent no peer will serve is never resolved, so without expiry it + /// occupies the budget permanently. + #[test] + fn unanswered_parent_request_expires_its_slot() { + let (mut pipeline, _tx, _progress_rx, _ctrl_rx, _data_rx, _dir) = test_pipeline(); + let now = std::time::Instant::now(); + + for i in 1..=3u8 { + assert!(pipeline.reorg_requested.should_request([i; 32], now)); + } + assert!(!pipeline.reorg_requested.should_request([4; 32], now)); + + let later = now + PARENT_REQUEST_TTL; + assert!( + pipeline.reorg_requested.should_request([4; 32], later), + "a request left unanswered past the TTL must release its slot" + ); + } } From 7a6bb774d1f2810185c1d74cbb3ae5b8806cf4d6 Mon Sep 17 00:00:00 2001 From: Muad'Dib Date: Tue, 18 Aug 2026 14:46:35 +0200 Subject: [PATCH 2/3] fix(validation,mining): expose nine CONTEXT.headers, not ten MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consensus divergence, accept-side. The JVM's `lastHeaders` holds ten and includes the block's own header at the head; `sigmaLastHeaders` — what a script sees as CONTEXT.headers — is `lastHeaders.drop(1)`, so nine. `UpcomingStateContext` overrides that with the whole list, which is why candidate and mempool prediction saw ten. Our window holds headers strictly preceding the block, since the block's own header lives in the preheader. So the JVM's drop(1) never meant "drop something here" — it meant take nine instead of ten. We took ten everywhere. `headerChainBack(10, …)` was cited in the source as parity evidence; it gathers lastHeaders, not sigmaLastHeaders, and that citation is what made ten look right. A script reading CONTEXT.headers(9) evaluated here and threw ArrayIndexOutOfBoundsException on every JVM node — we would follow a chain the network orphans, with nothing in the logs reporting it. Surfaced by the mainnet block-production incident of 2026-08-18; resolution agreed cross-client as nine plus preheader on every path. Co-Authored-By: Claude Opus 5 (1M context) --- facts/mining.md | 13 ++- facts/validation.md | 75 ++++++++++++++ mining/src/lib.rs | 174 ++++++++++++++++++++++++++++++-- src/main.rs | 17 +++- validation/src/lib.rs | 5 +- validation/src/tx_validation.rs | 134 +++++++++++++++++------- 6 files changed, 366 insertions(+), 52 deletions(-) diff --git a/facts/mining.md b/facts/mining.md index 178a767..a4d7af2 100644 --- a/facts/mining.md +++ b/facts/mining.md @@ -482,11 +482,16 @@ crate owns the assembly: transaction against a context built by `build_state_context(stub, preceding_headers, parameters)`. Built from the parent alone that context exposes a **one-header** `CONTEXT.headers` window, where block validation -exposes up to ten. A script reading `headers[5]` would then fail *during +exposes nine. A script reading `headers[5]` would then fail *during selection* and the transaction would be evicted from the mempool as invalid — -a valid transaction destroyed by a selection-only artefact. Pass -`chain.headers_from(parent.height - 9, 10)` reversed, minus the parent; the -parent is prepended internally. +a valid transaction destroyed by a selection-only artefact. + +⚠ **The window is nine, not ten** — see `facts/validation.md` § "Window size: +`CONTEXT.headers` is 9 for a block, never 10". Selection must predict with the +same window block validation judges with, or it packs transactions the block +that carries them will be rejected for. Pass +`chain.headers_from(parent.height - 8, 9)` reversed, minus the parent; the +parent is prepended internally, giving nine total. `generate_candidate` returns `GeneratedCandidate { block, work, invalid_txs }`. ⚠ **`invalid_txs` MUST be routed to the mempool for eviction** or Step 3.6 is diff --git a/facts/validation.md b/facts/validation.md index 9cb8540..6be8d9a 100644 --- a/facts/validation.md +++ b/facts/validation.md @@ -602,6 +602,81 @@ pub fn build_upcoming_state_context( ) -> ErgoStateContext; ``` +### Window size: `CONTEXT.headers` is 9 for a block, never 10 + +**`build_state_context` must expose exactly 9 preceding headers.** This is a +consensus rule, not a tuning choice. + +The JVM keeps two different things under similar names, and conflating them is +what produced the 2026-08-18 divergence: + +| JVM name | Size | What it is | +|---|---|---| +| `lastHeaders` | 10 | internal list, **includes the block's own header at the head** | +| `sigmaLastHeaders` | **9** | what a script sees as `CONTEXT.headers` | + +`ErgoStateContext.scala`: + +- L233 — `newHeaders = header +: lastHeaders.take(LastHeadersInContext - 1)`. + Appending block `B` yields `[B, h-1 … h-9]`: ten entries, `B` at the head. +- L87 — base class: `sigmaLastHeaders = lastHeaders.drop(1)` → `[h-1 … h-9]`, + **nine**. The dropped entry is `B` itself. +- L46 — `UpcomingStateContext` overrides `sigmaLastHeaders` with the whole + `lastHeaders`, **no drop** → ten. There is no block of its own to drop. + +Our builders take headers **strictly preceding** the block — the block's own +header goes in the preheader, never in the window. So the JVM's `drop(1)` does +not translate to dropping anything on our side; it translates to **taking nine +instead of ten**. `headerChainBack(10, …)` gathers `lastHeaders`, not +`sigmaLastHeaders`, and citing it as parity evidence for a 10-header window is +the specific error to avoid. + +#### Caller obligations + +| Path | `CONTEXT.headers` | Consensus | +|---|---|---| +| block validation (`build_state_context`) | **9** | **yes** | +| mining candidate assembly | 9 | no | +| mempool / API (`build_upcoming_state_context`) | 9 | no | + +Only the first is consensus. The other two are set to 9 **deliberately**, so a +transaction can never be admitted to the mempool or packed into a candidate and +then rejected by the block validation that must follow it. A path that predicts +with a wider window than the one that judges is the exact shape of the JVM +incident below. + +⚠ **This is convergence with the JVM, not a divergence from it.** The agreed +resolution is nine on every path — kushti, 2026-08-18: *"There must be 9 plus +preheader everywhere."* The JVM is aligning `UpcomingStateContext` down to nine +rather than widening full-block validation to ten, which would have needed +coordinated protocol activation. + +Until that lands, a transaction reading `CONTEXT.headers(9)` is still accepted +by a JVM mempool and refused by ours. Ours is the safe side of a transient gap: +that transaction cannot be mined into a block any node will accept, so refusing +it early costs nothing. + +#### Why this matters + +A script reading `CONTEXT.headers(9)` or branching on `CONTEXT.headers.size` +sees a different chain depending on which client validates it. With a 10-header +window we **accept a block every JVM node rejects** with +`ArrayIndexOutOfBoundsException`, and follow a chain the network orphans. The +divergence is accept-side, which is the dangerous direction: nothing in our logs +reports a problem. + +Observed on mainnet 2026-08-18 (JVM block-production incident at ~1853471–4, +reported by kushti): a script at `b44970ed…` reads `headers(9)`, so it passed +JVM candidate construction at ten and threw on the completed block at nine. Our +node was 10/10/10 — internally consistent, so it could not hit the JVM's +candidate-vs-block failure, but its full-block validation was one header more +permissive than consensus. It was spared only because it never obtained the +block body in question. + +`sigma-rust` is **not** the constraint: `Headers = BoundedVec` +permits one through ten and enforces nothing about which. The window size is +entirely the caller's contract. + ### Why two An unconfirmed transaction is not a member of the chain tip — it is a candidate diff --git a/mining/src/lib.rs b/mining/src/lib.rs index bce4634..8d2b89c 100644 --- a/mining/src/lib.rs +++ b/mining/src/lib.rs @@ -83,9 +83,12 @@ pub struct GeneratedCandidate { /// `max_block_size`. Not `boundary_params`: those are the parameters the /// epoch boundary will *install*, and the JVM likewise bounds assembly with /// `stateContext.currentParameters` (`CandidateGenerator.scala:591-598`) -/// - `ancestor_headers` — headers before `parent`, newest first. `parent` is -/// prepended internally to form the ≤10-header window the upcoming block's -/// `ErgoStateContext` needs, so this may be empty near genesis +/// - `ancestor_headers` — headers before `parent`, newest first, as +/// `chain.headers_from(parent.height - 8, 9)` reversed minus the parent. +/// `parent` is prepended internally to form the NINE-header +/// `CONTEXT.headers` window the upcoming block's `ErgoStateContext` needs +/// (see `context_window` for why nine and not ten), so at most eight of +/// these are used and near genesis the slice may be short or empty /// - `utxo_lookup` — resolve a box id against the UTXO set /// /// `validator_proofs` is a closure that calls @@ -134,9 +137,7 @@ pub fn generate_candidate( (vec![emission_tx], Vec::new()) } else { let upcoming = upcoming_header(parent, version, n_bits, height, timestamp, config); - let mut window = Vec::with_capacity(1 + ancestor_headers.len().min(9)); - window.push(parent.clone()); - window.extend(ancestor_headers.iter().take(9).cloned()); + let window = context_window(parent, ancestor_headers); let state_context = build_state_context(&upcoming, &window, parameters); select_and_collect_fees( @@ -192,6 +193,45 @@ pub fn generate_candidate( }) } +/// Headers a script sees as `CONTEXT.headers` — **nine**, the JVM's +/// `sigmaLastHeaders`, not the ten-entry `lastHeaders` it is derived from +/// (that list carries the block's own header at its head, and ours never +/// does: the block's header goes in the preheader). +/// +/// `facts/validation.md` § "Window size: `CONTEXT.headers` is 9 for a block, +/// never 10" is the authority. Restated here rather than imported because +/// nothing exports it yet; it belongs next to `build_state_context` once +/// `ergo-validation` names it. +const CONTEXT_HEADERS: usize = 9; + +/// The `CONTEXT.headers` window the candidate's transactions execute under: +/// `parent` at the head, then up to eight of its ancestors, newest first — +/// `CONTEXT_HEADERS` total. +/// +/// Selection validates every candidate transaction against this window, so it +/// must be the window block validation will judge with. A path that predicts +/// with a wider one eventually packs a transaction into a block that cannot +/// be accepted: the JVM shipped exactly that inconsistency and it took +/// mainnet block production down on 2026-08-18 — a script reading +/// `headers(9)` passed candidate construction at ten and threw on the +/// completed block at nine. +/// +/// The count is cut here rather than left to `build_state_context`'s own +/// truncation. A window that is only the right size because someone else +/// trims it is the next person's bug. +/// +/// ⚠ Near genesis fewer than eight ancestors is legal chain state — the JVM's +/// `headerChainBack` stops at genesis — so a short window is passed through +/// unpadded. Padding by repeating the oldest header diverged from the +/// reference node once already. +fn context_window(parent: &Header, ancestor_headers: &[Header]) -> Vec
{ + let max_ancestors = CONTEXT_HEADERS - 1; + let mut window = Vec::with_capacity(1 + ancestor_headers.len().min(max_ancestors)); + window.push(parent.clone()); + window.extend(ancestor_headers.iter().take(max_ancestors).cloned()); + window +} + /// The header the candidate's transactions will execute under. /// /// Only the `PreHeader` fields matter — that is all `build_state_context` @@ -619,3 +659,125 @@ impl CandidateGenerator { } } } + +#[cfg(test)] +mod tests { + use super::*; + use ergo_chain_types::EcPoint; + use ergo_validation::build_state_context; + + /// A syntactically complete header at `height`, distinguishable by `seed`. + /// Nothing here is chain-valid — the window is a slice operation and never + /// inspects linkage. + fn header_at(height: u32, seed: u8) -> Header { + Header { + version: 2, + id: BlockId(Digest::from([seed; 32])), + parent_id: BlockId(Digest::from([seed.wrapping_sub(1); 32])), + ad_proofs_root: Digest32::from([0u8; 32]), + state_root: ADDigest::from([0u8; 33]), + transaction_root: Digest32::from([0u8; 32]), + timestamp: 1_000 + u64::from(height), + n_bits: 16842752, + height, + extension_root: Digest32::from([0u8; 32]), + autolykos_solution: AutolykosSolution { + miner_pk: Box::new(EcPoint::default()), + pow_onetime_pk: None, + nonce: vec![0u8; 8], + pow_distance: None, + }, + votes: Votes([0, 0, 0]), + unparsed_bytes: Box::new([]), + } + } + + /// A parent at `parent_height` and the `count` headers below it, newest + /// first — the shape `generate_candidate`'s caller passes. + fn parent_and_ancestors(parent_height: u32, count: u32) -> (Header, Vec
) { + let parent = header_at(parent_height, parent_height as u8); + let ancestors = (1..=count) + .map(|back| header_at(parent_height - back, (parent_height - back) as u8)) + .collect(); + (parent, ancestors) + } + + /// `CONTEXT.headers` is NINE for a block, never ten + /// (`facts/validation.md`). Selection must predict with the window block + /// validation judges with, or it packs transactions into a block that + /// cannot be accepted. + /// + /// The count is asserted on the window this crate builds, not only on the + /// context that comes out of it: `build_state_context` truncates too, and + /// a window that is only the right size because someone else cut it is + /// the next person's bug. + #[test] + fn context_window_is_nine_headers_when_ancestors_are_plentiful() { + let (parent, ancestors) = parent_and_ancestors(100, 12); + let window = context_window(&parent, &ancestors); + + assert_eq!( + window.len(), + 9, + "window must be the parent plus EIGHT ancestors; got heights {:?}", + window.iter().map(|h| h.height).collect::>() + ); + assert_eq!(window[0].id, parent.id, "the parent heads the window"); + assert_eq!( + window[1..].iter().map(|h| h.height).collect::>(), + (92..=99).rev().collect::>(), + "ancestors follow newest first, contiguous below the parent" + ); + assert!( + !window.iter().any(|h| h.height == 91), + "the ninth ancestor is past the window and must be dropped here, \ + not left for build_state_context to cut" + ); + + let upcoming = Header { + height: parent.height + 1, + parent_id: parent.id, + ..parent.clone() + }; + let ctx = build_state_context(&upcoming, &window, &Parameters::default()); + assert_eq!( + ctx.headers.len(), + 9, + "CONTEXT.headers as a selected transaction sees it" + ); + } + + /// Near genesis fewer than eight ancestors is legal chain state — the + /// JVM's `headerChainBack` stops at genesis. Pass the short window + /// through unpadded; padding by repeating the oldest header diverged from + /// the reference node once already. + #[test] + fn context_window_near_genesis_is_short_and_unpadded() { + // Genesis is height 1 in Ergo: mining the block above it has no + // ancestors to offer at all. + let genesis = header_at(1, 1); + assert_eq!( + context_window(&genesis, &[]).len(), + 1, + "at genesis the parent is the whole window" + ); + + // Height 4: three headers exist below the parent, not eight. + let (parent, ancestors) = parent_and_ancestors(4, 3); + let window = context_window(&parent, &ancestors); + assert_eq!( + window.len(), + 4, + "parent plus the three ancestors that exist, not padded to nine" + ); + assert_eq!( + window.iter().map(|h| h.height).collect::>(), + vec![4, 3, 2, 1], + "the real chain, newest first, down to genesis" + ); + let mut ids: Vec<_> = window.iter().map(|h| h.id).collect(); + ids.sort_by_key(|id| id.0); + ids.dedup(); + assert_eq!(ids.len(), 4, "no header is repeated to pad the window"); + } +} diff --git a/src/main.rs b/src/main.rs index ded89b3..13783d2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4045,14 +4045,21 @@ async fn main() -> Result<(), Box> { }; // Ancestors for the upcoming-block context, newest first, - // WITHOUT the parent — generate_candidate prepends it. A - // one-header window here would fail any script reading - // headers[5] and get a valid transaction evicted. + // WITHOUT the parent — generate_candidate prepends it, + // giving the 9 of `CONTEXT.headers`. A one-header window + // here would fail any script reading headers[5] and get a + // valid transaction evicted. + // + // 9 headers in, minus the parent, is the 8 ancestors + // `context_window` keeps. Fetching 10 also works — mining + // truncates — but supplying a window that is only the right + // size because the callee trims it is how the 10-vs-9 + // divergence hid. See `facts/validation.md` § "Window size". let (active_params, ancestor_headers) = { let chain_guard = mining_chain.lock().await; let params = chain_guard.active_parameters().clone(); - let mut hs = chain_guard - .headers_from(proof_data.parent.height.saturating_sub(9), 10); + let mut hs = + chain_guard.headers_from(proof_data.parent.height.saturating_sub(8), 9); hs.reverse(); hs.retain(|h| h.height != proof_data.parent.height); (params, hs) diff --git a/validation/src/lib.rs b/validation/src/lib.rs index bc1f9a6..e2610a3 100644 --- a/validation/src/lib.rs +++ b/validation/src/lib.rs @@ -74,7 +74,10 @@ pub struct ScriptEvalInputs { pub proof_boxes: HashMap<[u8; 32], ErgoBox>, /// Block header. pub header: Header, - /// Up to 10 preceding headers (newest first). + /// Preceding headers, newest first. `build_state_context` exposes the + /// newest **9** of these as `CONTEXT.headers` — the JVM's + /// `sigmaLastHeaders`, not its ten-entry `lastHeaders`. Supplying more is + /// harmless; supplying fewer near genesis is legal chain state. pub preceding_headers: Vec
, /// Active chain parameters. pub parameters: Parameters, diff --git a/validation/src/tx_validation.rs b/validation/src/tx_validation.rs index 223de0c..2a3d617 100644 --- a/validation/src/tx_validation.rs +++ b/validation/src/tx_validation.rs @@ -32,28 +32,52 @@ pub fn deserialize_box(bytes: &[u8]) -> Result { }) } -/// Build an ErgoStateContext from a header and its real ≤10 preceding headers -/// (newest first), passed through unpadded. +/// Build an ErgoStateContext from a header and its preceding headers (newest +/// first). Exposes **exactly 9** of them as `CONTEXT.headers`; a shorter window +/// passes through unpadded. /// -/// The JVM gathers the same variable window — `headerChainBack(10, …)` stops -/// at genesis (`FullBlockProcessor:71`), so fewer than 10 headers near genesis -/// is legal chain state; padding to 10 by repeating the oldest (what this did -/// before) diverged from the reference node for scripts reading -/// `CONTEXT.headers`. ergo-lib derives `lastBlockUtxoRoot` from the newest -/// preceding header's state_root — identical to the JVM's -/// `previousStateDigest` (`ErgoStateContext.scala:92`) — with AvlTree flags -/// verified against `ErgoContext.scala:17` / `ErgoInterpreter.scala:103-106`; -/// nothing to override on our side. +/// # Nine, never ten — this one is consensus /// -/// Requires ≥ 1 preceding header — caller-guarded, and now also enforced by -/// the `Headers` type. +/// The JVM keeps two lists under similar names, and conflating them forks the +/// chain (`ErgoStateContext.scala`): +/// +/// - `lastHeaders` — ten, and **includes the block's own header at the head**: +/// `newHeaders = header +: lastHeaders.take(LastHeadersInContext - 1)` (L233). +/// - `sigmaLastHeaders` — what a script sees as `CONTEXT.headers`: +/// `lastHeaders.drop(1)` (L87), so **nine**. The dropped entry is the block +/// itself. +/// +/// Our window holds headers *strictly preceding* the block — the block's own +/// header goes in the preheader and is never in this slice — so the JVM's +/// `drop(1)` does not mean "drop something here". It means **take nine instead +/// of ten**. `headerChainBack(10, …)` (`FullBlockProcessor:71`) gathers +/// `lastHeaders`, not `sigmaLastHeaders`; citing it as parity evidence for a +/// ten-header window is the specific error that produced the divergence this +/// replaces. At ten we accept a block every JVM node rejects with +/// `ArrayIndexOutOfBoundsException` on `CONTEXT.headers(9)` and follow a chain +/// the network orphans — accept-side, so nothing in our logs reports it. +/// (Observed on mainnet 2026-08-18: script `b44970ed…` reads `headers(9)`.) +/// +/// Fewer than nine near genesis is legal chain state — `headerChainBack` stops +/// there — and passes through unpadded; padding by repeating the oldest (what +/// this did before that) is its own divergence. `sigma-rust` enforces nothing +/// here: `Headers = BoundedVec` permits any of one through ten. +/// +/// ergo-lib derives `lastBlockUtxoRoot` from the newest preceding header's +/// state_root — identical to the JVM's `previousStateDigest` +/// (`ErgoStateContext.scala:92`) — with AvlTree flags verified against +/// `ErgoContext.scala:17` / `ErgoInterpreter.scala:103-106`; nothing to +/// override on our side. +/// +/// Requires ≥ 1 preceding header — caller-guarded, and also enforced by the +/// `Headers` type. pub fn build_state_context( header: &Header, preceding_headers: &[Header], parameters: &Parameters, ) -> ErgoStateContext { let pre_header = PreHeader::from(header.clone()); - let headers = Headers::from_vec(preceding_headers.iter().take(10).cloned().collect()) + let headers = Headers::from_vec(preceding_headers.iter().take(9).cloned().collect()) .expect("build_state_context requires at least one preceding header (caller-guarded)"); ErgoStateContext::new(pre_header, headers, parameters.clone()) } @@ -89,7 +113,8 @@ pub fn build_state_context( /// Unlike `build_state_context` there is no non-empty requirement: /// `preceding_headers` may be empty (height 1), since `last_header` alone /// already satisfies the `Headers` lower bound. The window is `last_header` -/// plus up to 9 of them — 10 total, the JVM's `LastHeadersInContext`. +/// plus up to 8 of them — **9 total, the same as block validation**, the tip +/// having taken one of the nine slots. /// /// # JVM reference /// @@ -97,11 +122,24 @@ pub fn build_state_context( /// composed with `PreHeader.apply` (`PreHeader.scala:49-63`) and /// `AutolykosPowScheme.derivedHeaderFields` (`AutolykosPowScheme.scala:455`). /// `UpcomingStateContext` overrides `sigmaLastHeaders` to the whole -/// `lastHeaders` with no `drop(1)` (`ErgoStateContext.scala:46`) — that missing -/// drop is precisely why the tip stays in the window here while block -/// validation excludes its own header. +/// `lastHeaders` with no `drop(1)` (`ErgoStateContext.scala:46`) — there is no +/// block of its own to drop, which is why the tip stays *in* the window here +/// while block validation keeps the block's own header out of it. /// -/// Two deliberate divergences, both recorded in the contract: +/// Three deliberate divergences, all recorded in the contract: +/// - the window is **9, where the JVM currently has 10**. That same missing +/// `drop(1)` leaves `UpcomingStateContext` one header wider than the block +/// validation which must follow it — the exact shape of the 2026-08-18 +/// mainnet incident, where a script reading `headers(9)` passed JVM candidate +/// construction at ten and threw on the completed block at nine. The agreed +/// cross-client resolution is nine everywhere (kushti, 2026-08-18: "There +/// must be 9 plus preheader everywhere"), with the JVM aligning `Upcoming` +/// down rather than block validation widening to ten — widening is a +/// consensus change and would need coordinated activation. Until that lands, +/// a transaction reading `headers(9)` is accepted by a JVM mempool and +/// refused here; that is the safe side of a transient gap, since no such +/// transaction can be mined into a block any node accepts. **Do not "fix" +/// this back to 10.** /// - `parameters` are the caller's, active for `last_header`. The JVM /// recomputes them for `height + 1` inside `simplifiedUpcoming()`; the two /// differ only on a block that crosses an epoch boundary. @@ -146,9 +184,9 @@ pub fn build_upcoming_state_context( votes: Votes([0, 0, 0]), }; - let mut window = Vec::with_capacity(1 + preceding_headers.len().min(9)); + let mut window = Vec::with_capacity(1 + preceding_headers.len().min(8)); window.push(last_header.clone()); - window.extend(preceding_headers.iter().take(9).cloned()); + window.extend(preceding_headers.iter().take(8).cloned()); let headers = Headers::from_vec(window).expect("window always holds last_header, so it is never empty"); @@ -1101,17 +1139,29 @@ mod state_context_window_tests { assert_eq!(ctx.headers.last().height, 1); } - /// Steady state: a full 10-header window passes through as-is. + /// Steady state: `CONTEXT.headers` is **nine**, however many the caller + /// supplies. + /// + /// Nine is the JVM's `sigmaLastHeaders` (`ErgoStateContext.scala:87` — + /// `lastHeaders.drop(1)`), not the ten-entry `lastHeaders`. The dropped + /// entry is the block's own header, which lives in our preheader and was + /// never in this slice, so the `drop(1)` translates to taking nine here + /// rather than to dropping anything. Twelve are passed in so the nine is + /// proven to be a cap and not merely "all of them". #[test] - fn full_window_passes_through_identically() { + fn window_is_nine_not_ten() { let header = header_at(20); - // Heights 19 down to 10, newest first. - let preceding: Vec
= (10..20).rev().map(header_at).collect(); + // Heights 19 down to 8, newest first — more than the window holds. + let preceding: Vec
= (8..20).rev().map(header_at).collect(); let ctx = build_state_context(&header, &preceding, &Parameters::default()); - assert_eq!(ctx.headers.len(), 10); - assert_eq!(ctx.headers.first().height, 19); - assert_eq!(ctx.headers.last().height, 10); + assert_eq!( + ctx.headers.len(), + 9, + "a tenth header is one every JVM node throws on indexing" + ); + assert_eq!(ctx.headers.first().height, 19, "the block's parent"); + assert_eq!(ctx.headers.last().height, 11, "nine back, not ten"); } } @@ -1246,15 +1296,22 @@ mod upcoming_state_context_tests { ); } - /// The tip occupies a slot in the window, so ten ancestors become nine. - /// Total stays at the JVM's `LastHeadersInContext`. + /// The tip occupies a slot in the nine, so only eight ancestors fit. + /// + /// Nine here is deliberate convergence rather than a copy of the JVM as it + /// stands: `UpcomingStateContext` overrides `sigmaLastHeaders` to the whole + /// ten-entry `lastHeaders` with no `drop(1)` (`ErgoStateContext.scala:46`), + /// and is being aligned *down* to nine rather than block validation being + /// widened to ten. A path that predicts against a wider window than the one + /// that judges is exactly the JVM's 2026-08-18 candidate-vs-block incident. #[test] - fn window_is_the_tip_plus_nine_ancestors() { + fn window_is_the_tip_plus_eight_ancestors() { let last = header_at(20); - let preceding: Vec
= (10..20).rev().map(header_at).collect(); + // Heights 19 down to 8 — more ancestors than the window holds. + let preceding: Vec
= (8..20).rev().map(header_at).collect(); let ctx = build_upcoming_state_context(&last, &preceding, &Parameters::default()); - assert_eq!(ctx.headers.len(), 10); + assert_eq!(ctx.headers.len(), 9, "nine plus preheader, everywhere"); assert_eq!( ctx.headers.first().height, 20, @@ -1262,8 +1319,8 @@ mod upcoming_state_context_tests { ); assert_eq!( ctx.headers.last().height, - 11, - "the tenth ancestor (height 10) drops out to make room for the tip" + 12, + "the tip takes a slot, so the ninth ancestor drops out" ); } @@ -1300,7 +1357,7 @@ mod upcoming_state_context_tests { let upcoming = build_upcoming_state_context(&last, &ancestors, &Parameters::default()); - // The block at 21, once mined, validates against the tip and its nine + // The block at 21, once mined, validates against the tip and its eight // ancestors — which is exactly what the mempool saw a moment earlier. let mut preceding_for_next = vec![last.clone()]; preceding_for_next.extend(ancestors.iter().cloned()); @@ -1317,6 +1374,11 @@ mod upcoming_state_context_tests { at_next.headers.as_vec(), "same chain position, same window" ); + // Equality alone is satisfied by two windows that are equally wrong — + // 10/10 was "internally consistent" too, and that is what let the + // divergence sit unseen. Pin the absolute size, not just the agreement. + assert_eq!(upcoming.headers.len(), 9); + assert_eq!(at_next.headers.len(), 9); } } From e60768eae3430d0868f2d3eb40ae9a4a6aa17bd1 Mon Sep 17 00:00:00 2001 From: Muad'Dib Date: Tue, 18 Aug 2026 14:46:35 +0200 Subject: [PATCH 3/3] chore: release v0.8.1 Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 68 ++++++++++++++++++++++++++++++++++++++++++++++ Cargo.lock | 20 +++++++------- Cargo.toml | 2 +- facts/openapi.yaml | 2 +- 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 311e8c9..830c987 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,73 @@ # Changelog +## v0.8.1 — 2026-08-18 + +### Release summary + +- Fixed: `CONTEXT.headers` exposed ten preceding headers where the reference node exposes nine, so a block every JVM node rejects could be accepted here. +- Fixed: the mempool and mining-candidate contexts also exposed ten; all three paths now expose nine. +- Fixed: a node that had spent its unknown-parent request budget could never ask for a missing parent again, leaving it stuck on an orphaned tip. + +### Fixed + +#### `CONTEXT.headers` exposes nine headers, not ten — consensus + +The JVM keeps two things under similar names. `lastHeaders` holds ten entries +and **includes the block's own header at the head** +(`newHeaders = header +: lastHeaders.take(LastHeadersInContext - 1)`). +`sigmaLastHeaders` — what a script actually sees as `CONTEXT.headers` — is +`lastHeaders.drop(1)`, so **nine**; the dropped entry is the block itself. +`UpcomingStateContext` overrides that with the whole list, which is why +candidate assembly and mempool prediction saw ten. + +Our window holds headers *strictly preceding* the block, because the block's +own header goes in the preheader and is never in the slice. So the JVM's +`drop(1)` never meant "drop something here" — it meant **take nine instead of +ten**. We took ten, on all three paths. `headerChainBack(10, …)` was cited in +the source as parity evidence; it gathers `lastHeaders`, not +`sigmaLastHeaders`, and that citation is what made ten look correct. + +A script reading `CONTEXT.headers(9)` therefore evaluated fine here and threw +`ArrayIndexOutOfBoundsException` on every JVM node. The divergence was +accept-side — this node would follow a chain the network orphans, with nothing +in its logs reporting a problem. + +Found when the same asymmetry took mainnet block production down on +2026-08-18: on the JVM, candidate construction validated at ten and the +completed block at nine, so a script reading `headers(9)` passed the first and +failed the second, and the transaction was pushed back into the mempool and +re-selected indefinitely. This node was self-consistent at ten across all three +paths and so could not hit that failure mode, but its block validation was one +header more permissive than consensus. All three paths are now nine, matching +the agreed cross-client resolution. + +**No resync is required.** A canonical block cannot contain a script the +reference node rejects, so nothing this node accepted depended on the tenth +header. A script that merely read `CONTEXT.headers.size` would have produced a +different state root and failed the existing state-root check loudly; that has +not happened. + +#### Unknown-parent requests are an in-flight limit, not a lifetime budget + +When a header arrives whose parent is unknown, the validation pipeline buffers +it and asks peers for the parent, bounded to three concurrent requests so a +batch of orphans cannot fan out into a request storm. The set backing that +bound was only ever inserted into — never cleared when a parent arrived, never +expired when none ever did. Three was therefore a budget for the lifetime of +the process, not a limit on requests in flight. + +Once spent, the node could no longer ask for a missing parent at all. It then +recovered only if some peer volunteered the header unprompted, which is a +matter of which peers it happens to be connected to. + +Observed on 2026-08-18: a node that woke from suspend holding an orphaned tip +at height 1,853,471 sat at that height for 1h48m — receiving headers it could +not attach and discarding them — until a peer finally announced the one header +it was no longer able to request. + +Request slots are now released when the parent arrives, and expire after 60 +seconds when it never does. + ## v0.8.0 — 2026-08-13