Skip to content

refactor(spice): tidy chunk-certification endorsement bookkeeping (1/6) #15936

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
darioush wants to merge 2 commits into
base: master
Choose a base branch
from
Open

refactor(spice): tidy chunk-certification endorsement bookkeeping (1/6) #15936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c7e4269
refactor(spice): extract endorsement-threshold tally to shared helper
darioush Jun 16, 2026
5c85509
refactor(spice): collect block-validation endorsements via AncestryEn…
darioush Jun 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions chain/chain/src/spice/ancestry_endorsements.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
use near_primitives::spice::chunk_endorsement::SpiceStoredVerifiedEndorsement;
use near_primitives::types::{AccountId, SpiceChunkId, SpiceUncertifiedChunkInfo};
use std::collections::{HashMap, HashSet};

/// Endorsement state derived from the uncertified chunks as of the previous block: the context
/// against which a block's core statements are validated. Borrows `prev_uncertified_chunks`.
pub(crate) struct AncestryEndorsements<'a> {
pending_designated: HashSet<(&'a SpiceChunkId, &'a AccountId)>,
on_chain: HashMap<&'a SpiceChunkId, HashMap<&'a AccountId, &'a SpiceStoredVerifiedEndorsement>>,
}

impl<'a> AncestryEndorsements<'a> {
pub(crate) fn collect(prev_uncertified_chunks: &'a [SpiceUncertifiedChunkInfo]) -> Self {
let mut on_chain: HashMap<_, HashMap<_, _>> = HashMap::new();
for info in prev_uncertified_chunks {
for (account_id, endorsement) in info.all_present_endorsements() {
on_chain.entry(&info.chunk_id).or_default().insert(account_id, endorsement);
}
}
Self {
pending_designated: prev_uncertified_chunks
.iter()
.flat_map(|info| {
info.missing_endorsements.iter().map(|account_id| (&info.chunk_id, account_id))
})
.collect(),
on_chain,
}
}

/// Whether `(chunk_id, account_id)` is a designated endorsement still awaited on chain.
pub(crate) fn is_pending_designated(
&self,
chunk_id: &'a SpiceChunkId,
account_id: &'a AccountId,
) -> bool {
self.pending_designated.contains(&(chunk_id, account_id))
}

/// Chunks awaiting a designated endorsement on chain (with repeats per missing validator).
pub(crate) fn pending_designated_chunks(&self) -> impl Iterator<Item = &'a SpiceChunkId> + '_ {
self.pending_designated.iter().map(|(chunk_id, _)| *chunk_id)
}

/// Endorsements (designated and non-designated) already on chain for `chunk_id`.
pub(crate) fn on_chain_for(
&self,
chunk_id: &SpiceChunkId,
) -> impl Iterator<Item = (&'a AccountId, &'a SpiceStoredVerifiedEndorsement)> + '_ {
self.on_chain
.get(chunk_id)
.into_iter()
.flatten()
.map(|(account_id, endorsement)| (*account_id, *endorsement))
}
}
147 changes: 87 additions & 60 deletions chain/chain/src/spice/core.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use crate::spice::ancestry_endorsements::AncestryEndorsements;
use crate::{Chain, ChainStoreAccess, ChainStoreUpdate};
use near_chain_primitives::Error;
use near_crypto::Signature;
Expand All @@ -11,8 +12,9 @@ use near_primitives::hash::CryptoHash;
use near_primitives::merkle::merklize;
use near_primitives::shard_layout::ShardUId;
use near_primitives::spice::chunk_endorsement::{
SpiceEndorsementCoreStatement, SpiceStoredVerifiedEndorsement,
SpiceEndorsementCoreStatement, SpiceEndorsementSignedData, SpiceStoredVerifiedEndorsement,
};
use near_primitives::stateless_validation::validator_assignment::ChunkValidatorAssignments;
use near_primitives::types::chunk_extra::ChunkExtra;
use near_primitives::types::validator_stake::ValidatorStake;
use near_primitives::types::{
Expand Down Expand Up @@ -75,6 +77,33 @@ impl SpiceCoreReader {
.get_ser(DBCol::endorsements(), &get_endorsements_key(block_hash, shard_id, account_id))
}

/// Whether the union of `endorsers` and the validators whose stored endorsement attests
/// `result_hash` certifies the chunk under `assignment`. `endorsers` seeds the set with the
/// endorsers already in hand (e.g. from the block); the rest are read from the store.
pub(crate) fn reaches_endorsement_threshold(
&self,
chunk_id: &SpiceChunkId,
result_hash: &ChunkExecutionResultHash,
assignment: &ChunkValidatorAssignments,
mut endorsers: HashSet<AccountId>,
) -> bool {
for (account_id, _) in assignment.assignments() {
if endorsers.contains(account_id) {
continue;
}
let Some(stored) =
self.get_endorsement(&chunk_id.block_hash, chunk_id.shard_id, account_id)
else {
continue;
};
if stored.execution_result_hash != *result_hash {
continue;
}
endorsers.insert(account_id.clone());
}
assignment.is_endorsed(&endorsers)
}

fn get_execution_result(
&self,
block_header: &BlockHeader,
Expand Down Expand Up @@ -303,6 +332,24 @@ impl SpiceCoreReader {
Ok(())
}

/// Pushes, as endorsement core statements, the stored endorsements the producer holds for
/// `accounts`. Accounts whose endorsement isn't in the store are simply skipped.
fn push_stored_endorsements<'b>(
&self,
chunk_id: &SpiceChunkId,
accounts: impl IntoIterator<Item = &'b AccountId>,
core_statements: &mut Vec<SpiceCoreStatement>,
) {
for account_id in accounts {
if let Some(endorsement) =
self.get_endorsement(&chunk_id.block_hash, chunk_id.shard_id, account_id)
{
core_statements
.push(endorsement.into_core_statement(chunk_id.clone(), account_id.clone()));
}
}
}

pub fn core_statements_for_next_block(
&self,
block_header: &BlockHeader,
Expand All @@ -316,17 +363,11 @@ impl SpiceCoreReader {

let mut core_statements = Vec::new();
for chunk_info in uncertified_chunks {
for account_id in chunk_info.missing_endorsements {
if let Some(endorsement) = self.get_endorsement(
&chunk_info.chunk_id.block_hash,
chunk_info.chunk_id.shard_id,
&account_id,
) {
core_statements.push(
endorsement.into_core_statement(chunk_info.chunk_id.clone(), account_id),
);
}
}
self.push_stored_endorsements(
&chunk_info.chunk_id,
&chunk_info.missing_endorsements,
&mut core_statements,
);

let Some(execution_result) = self.get_execution_result_from_store(
&chunk_info.chunk_id.block_hash,
Expand Down Expand Up @@ -425,6 +466,20 @@ impl SpiceCoreReader {
Ok(())
}

/// Verifies `endorsement`'s signature against its signer's key in `epoch_id`, returning the
/// signed data and signature. The error is a reason string for `InvalidCoreStatement`.
fn verify_endorsement_signature<'e>(
&self,
epoch_id: &EpochId,
endorsement: &'e SpiceEndorsementCoreStatement,
) -> Result<(&'e SpiceEndorsementSignedData, &'e Signature), &'static str> {
let validator_info = self
.epoch_manager
.get_validator_by_account_id(epoch_id, endorsement.account_id())
.map_err(|_| "endorsement from non-validator")?;
endorsement.verified_signed_data(validator_info.public_key()).ok_or("invalid signature")
}

pub fn validate_core_statements_in_block(
&self,
block: &Block,
Expand All @@ -446,23 +501,7 @@ impl SpiceCoreReader {
tracing::debug!(target: "spice_core", prev_hash=?block.header().prev_hash(), ?err, "failed getting uncertified_chunks");
NoPrevUncertifiedChunks
})?;
let waiting_on_endorsements: HashSet<_> = prev_uncertified_chunks
.iter()
.flat_map(|info| {
info.missing_endorsements.iter().map(|account_id| (&info.chunk_id, account_id))
})
.collect();
let known_endorsements: HashMap<
(&SpiceChunkId, &AccountId),
&SpiceStoredVerifiedEndorsement,
> = prev_uncertified_chunks
.iter()
.flat_map(|info| {
info.present_endorsements
.iter()
.map(|(account_id, endorsement)| ((&info.chunk_id, account_id), endorsement))
})
.collect();
let ancestry_endorsements = AncestryEndorsements::collect(&prev_uncertified_chunks);

let mut in_block_endorsements: HashMap<
&SpiceChunkId,
Expand All @@ -478,11 +517,9 @@ impl SpiceCoreReader {
let account_id = endorsement.account_id();
// TODO(spice): reject more than one endorsement per (chunk, account),
// regardless of result hash. The dup check below is per result hash and
// `waiting_on_endorsements` is not updated mid-loop, so a validator can
// `pending_designated` is not updated mid-loop, so a validator can
// equivocate (endorse two results for one chunk) and count toward both.
// Checking contents of waiting_on_endorsements makes sure that
// chunk_id and account_id are valid.
if !waiting_on_endorsements.contains(&(chunk_id, account_id)) {
if !ancestry_endorsements.is_pending_designated(chunk_id, account_id) {
return Err(InvalidCoreStatement {
index,
// It can either be already included in the ancestry or be for a block
Expand All @@ -491,18 +528,14 @@ impl SpiceCoreReader {
});
}

let block = get_block(self.chain_store.store_ref(), &chunk_id.block_hash)?;

let validator_info = self
.epoch_manager
.get_validator_by_account_id(block.header().epoch_id(), account_id)
.expect("we are waiting on endorsement for this account so relevant validator has to exist");

let Some((signed_data, signature)) =
endorsement.verified_signed_data(validator_info.public_key())
else {
return Err(InvalidCoreStatement { index, reason: "invalid signature" });
};
let endorsement_block =
get_block(self.chain_store.store_ref(), &chunk_id.block_hash)?;
let (signed_data, signature) = self
.verify_endorsement_signature(
endorsement_block.header().epoch_id(),
endorsement,
)
.map_err(|reason| InvalidCoreStatement { index, reason })?;

if in_block_endorsements
.entry(chunk_id)
Expand Down Expand Up @@ -539,7 +572,7 @@ impl SpiceCoreReader {

// TODO(spice): Add validation that endorsements for blocks are included only when previous
// block is fully endorsed (as part of block we are validating or it's ancestry).
for (chunk_id, _) in &waiting_on_endorsements {
for chunk_id in ancestry_endorsements.pending_designated_chunks() {
if block_execution_results.contains_key(chunk_id) {
continue;
}
Expand All @@ -554,7 +587,7 @@ impl SpiceCoreReader {
// We cannot be waiting on an endorsement for chunk created at height that is less
// than maximum endorsed height for the chunk as that would mean that child is
// endorsed before parent.
return Err(SkippedExecutionResult { chunk_id: (*chunk_id).clone() });
return Err(SkippedExecutionResult { chunk_id: chunk_id.clone() });
}
}

Expand All @@ -570,19 +603,12 @@ impl SpiceCoreReader {
.expect(
"since we are waiting for endorsement we should know it's validator assignments",
);
for (account_id, _) in chunk_validator_assignments.assignments() {
// We cannot look for endorsements in store since there they are written by a
// separate actor which isn't synchronized with block processing.
if !waiting_on_endorsements.contains(&(chunk_id, account_id)) {
let endorsement = known_endorsements.get(&(chunk_id, account_id))
.expect(
"if we aren't waiting for endorsement in this block it should be in ancestry and known"
);
on_chain_endorsements
.entry(endorsement.execution_result_hash.clone())
.or_default()
.insert(account_id, endorsement.signature.clone());
}
// Add the on-chain ancestry endorsements to the in-block ones, grouped by attested result.
for (account_id, endorsement) in ancestry_endorsements.on_chain_for(chunk_id) {
on_chain_endorsements
.entry(endorsement.execution_result_hash.clone())
.or_default()
.insert(account_id, endorsement.signature.clone());
}
for (execution_result_hash, validator_signatures) in on_chain_endorsements {
let endorsement_state =
Expand Down Expand Up @@ -799,6 +825,7 @@ pub fn record_uncertified_chunks_for_block(
chunk_id: SpiceChunkId { block_hash: *block.hash(), shard_id },
missing_endorsements,
present_endorsements: Vec::new(),
present_fallback_endorsements: Vec::new(),
});
}

Expand Down
Loading
Loading