feat(forknet): combined early chunk producer kickout

CHANGELOG.md

@@ -15,6 +15,7 @@
 * Fix `action_delete_account` not accounting for the global contract identifier when checking the account deletion storage limit. For accounts using a global contract, the check overcounted storage usage by the identifier size (32 bytes, or the account id length for `GlobalByAccount`), so they were slightly harder to delete. ([#15752](https://github.com/near/nearcore/pull/15752))
 * Fix a bug in `receiver` verification for a `DeterministicStateInitAction` inside a `DelegateAction` that made it impossible to create deterministic accounts through meta transactions. ([#15812](https://github.com/near/nearcore/pull/15812))
 * Stabilized gas keys ([NEP-611](https://github.com/near/NEPs/pull/611)): a new access key type that carries a pre-funded NEAR balance and up to 1,024 independent nonce sequences. Transactions signed with a gas key pay gas costs from the key's balance rather than the signer's account balance. ([#15183](https://github.com/near/nearcore/pull/15183))
+* nightly mid-epoch chunk-producer kickout (gated at protocol v152): a chunk producer that stops producing is excluded from chunk-producer sampling within the same epoch, reassigning its slots to other producers instead of waiting for the epoch boundary. ([#15843](https://github.com/near/nearcore/pull/15843))
 * Stabilized the `promise_yield_create_with_id` and `promise_yield_resume_with_yield_id` host functions, which let a contract supply its own deterministic 32-byte yield ID when creating a yield/resume promise instead of storing the runtime-generated `data_id` in state. This avoids the storage cost of tracking yields, enabling use cases such as in-contract mempools. ([#15602](https://github.com/near/nearcore/pull/15602))
 * Increased the cost of creating new accounts from `~0.0008` NEAR to `~0.007 NEAR` ([NEP-642](https://github.com/near/NEPs/pull/642)), paid in gas. The gas will now be purchased at a higher price than before (at least `min_gas_purchase_price = 0.001 NEAR/TGas`) to cover the cost of creating new accounts. Gas will still be burned at the same price as before (usually `min_gas_price = 0.0001 NEAR/TGas`) and the price difference will be refunded, so the total cost of a transaction will stay the same. However the balance required to start a transaction will be 10x higher than before, as the price at which gas is purchased will be 10x higher than before. For any operation which creates an account, the protocol will deduct `~0.007 NEAR` from the gas refund to cover the cost of creating an account.
 * Meta transactions with gas key support via `Action::DelegateV2`. ([#15183](https://github.com/near/nearcore/pull/15183))

chain/chain/src/blocks_delay_tracker.rs

@@ -142,7 +142,7 @@ impl ChunkTrackingStats {
             ChunkProcessingStatus::NeedToRequest
         };
         let created_by = epoch_manager
-            .get_chunk_producer_info_db(&self.prev_block_hash, self.shard_id)
+            .get_chunk_producer_info_from_prev_block(&self.prev_block_hash, self.shard_id)
             .map(|info| info.take_account_id())
             .ok();
         let request_duration = if let Some(requested_timestamp) = self.requested_timestamp {

chain/chain/src/chain.rs

@@ -75,7 +75,6 @@ use near_primitives::sharding::{
     ChunkHash, ReceiptProof, ShardChunk, ShardChunkHeader, ShardProof, StateSyncInfo,
 };
 use near_primitives::state_sync::ReceiptProofResponse;
-use near_primitives::stateless_validation::ChunkProductionKey;
 use near_primitives::stateless_validation::state_witness::{
     ChunkStateWitness, ChunkStateWitnessSize,
 };
@@ -3584,15 +3583,17 @@ impl Chain {
         // Create the callback only when this node is the chunk producer for the next height. It's
         // used only for early prepare transactions, doesn't make sense to call it if the node isn't
         // a chunk producer.
-        let cpk = ChunkProductionKey {
-            shard_id: shard_uid.shard_id(),
-            epoch_id: epoch_id,
-            height_created: block.height + 1,
-        };
         let Some(signer) = self.validator_signer.get() else {
             return None;
         };
-        let Ok(producer) = self.epoch_manager.get_chunk_producer_info(&cpk) else {
+        // The next chunk's parent is `block`, so its grandparent anchor is
+        // `prev_block` — already processed, unlike `block` itself.
+        let Ok(producer) = self.epoch_manager.get_chunk_producer_info_anchored(
+            Some(prev_block.hash()),
+            &epoch_id,
+            block.height + 1,
+            shard_uid.shard_id(),
+        ) else {
             return None;
         };
         if signer.validator_id() != producer.account_id() {

chain/chain/src/genesis.rs

@@ -137,19 +137,13 @@ impl Chain {
                 .add_validator_proposals(block_info, *genesis.header().random_value())?
                 .into(),
         );
-        // Save chunk producers for height 1 (next height after genesis).
+        // Save chunk producers anchored at genesis (chunks at genesis + 2). Chunks at
+        // genesis + 1 and below have no grandparent and resolve via the canonical sampler.
         store_update.save_chunk_producers_for_header(
             epoch_manager,
             genesis.header(),
             genesis_protocol_version,
         )?;
-        // Save chunk producers for the genesis chunks themselves (height 0).
-        // Genesis chunks have prev_block_hash = CryptoHash::default().
-        store_update.save_genesis_chunk_producers(
-            epoch_manager,
-            genesis_protocol_version,
-            genesis.header().height(),
-        )?;
         store_update.save_block_header(genesis.header().clone())?;
         store_update.save_block(genesis.clone().into());
         Self::save_genesis_chunk_extras(&genesis, &state_roots, epoch_manager, &mut store_update)?;

chain/chain/src/runtime/tests.rs

@@ -214,6 +214,7 @@ impl TestEnv {
             )
             .unwrap()
             .commit();
+        epoch_manager.read().seed_chunk_producers_for_test(&genesis_hash);
         Self {
             epoch_manager,
             runtime,
@@ -386,6 +387,7 @@ impl TestEnv {
             )
             .unwrap()
             .commit();
+        self.epoch_manager.read().seed_chunk_producers_for_test(&new_hash);
         let shard_layout = self.epoch_manager.get_shard_layout_from_prev_block(&new_hash).unwrap();
         let mut new_receipts = HashMap::<_, Vec<Receipt>>::new();
         for receipt in all_receipts {
@@ -904,6 +906,7 @@ fn test_state_sync() {
             )
             .unwrap()
             .commit();
+        new_env.epoch_manager.read().seed_chunk_producers_for_test(&cur_hash);
         new_env.head.height = i;
         new_env.head.last_block_hash = cur_hash;
         new_env.head.prev_block_hash = prev_hash;

chain/chain/src/signature_verification.rs

@@ -24,10 +24,10 @@ pub fn verify_block_vrf(
     Ok(())
 }
 
-/// Verify chunk header signature using hash-based chunk producer lookup.
-/// Uses get_chunk_producer_info_db(prev_block_hash, shard_id) which reads from
-/// the ChunkProducers DB column when EarlyKickout is enabled, and errors on
-/// a missing DB entry.
+/// Verify chunk header signature using the anchored chunk producer lookup.
+/// Under EarlyKickout the producer is read from the ChunkProducers DB column
+/// keyed by the chunk's grandparent anchor; cross-epoch and low-height chunks,
+/// and the feature-off path, fall back to the canonical height sampler.
 pub fn verify_chunk_header_signature_by_hash(
     epoch_manager: &dyn EpochManagerAdapter,
     chunk_header: &ShardChunkHeader,
@@ -48,7 +48,8 @@ pub fn verify_chunk_header_signature_by_hash_and_parts(
     prev_block_hash: &CryptoHash,
     shard_id: ShardId,
 ) -> Result<bool, Error> {
-    let chunk_producer = epoch_manager.get_chunk_producer_info_db(prev_block_hash, shard_id)?;
+    let chunk_producer =
+        epoch_manager.get_chunk_producer_info_from_prev_block(prev_block_hash, shard_id)?;
     Ok(signature.verify(chunk_hash.as_ref(), chunk_producer.public_key()))
 }
 

chain/chain/src/store/mod.rs

@@ -4,7 +4,7 @@ use borsh::{BorshDeserialize, BorshSerialize};
 use chrono::Utc;
 pub use latest_witnesses::LatestWitnessesInfo;
 use near_chain_primitives::error::Error;
-use near_epoch_manager::EpochManagerAdapter;
+use near_epoch_manager::{CHUNK_GRANDPARENT_ANCHOR_HEIGHT_OFFSET, EpochManagerAdapter};
 use near_primitives::block::Tip;
 use near_primitives::chunk_apply_stats::{ChunkApplyStats, ChunkApplyStatsV1};
 use near_primitives::errors::{EpochError, InvalidTxError};
@@ -1829,14 +1829,23 @@ impl<'a> ChainStoreUpdate<'a> {
         self.gc_stop_height = Some(height);
     }
 
-    /// Pre-computes and persists chunk producer assignments for the block following `header`.
+    /// Pre-computes and persists chunk producer assignments anchored at `header`.
     ///
-    /// For each shard in the epoch after `header`, samples the chunk producer at
-    /// height `header.height() + 1` and writes it to `DBCol::ChunkProducers` keyed by
-    /// `(header.hash(), shard_id)`. This makes historical chunk producer lookups
-    /// available from the DB without recomputation.
+    /// `header` is the grandparent anchor of the chunks these rows resolve: for
+    /// each shard in the epoch after `header`, samples the chunk producer at
+    /// height `header.height() + 2` (the height of a chunk whose grandparent is
+    /// `header`, absent skips) and writes it to `DBCol::ChunkProducers` keyed by
+    /// `(header.hash(), shard_id)`. Sampling uses the epoch after `header`; a
+    /// chunk in a later epoch never reads this row (the cross-epoch arm of
+    /// `get_chunk_producer_info_anchored` routes it to the canonical sampler).
     ///
     /// Gated behind `EarlyKickout` protocol feature. No-op when disabled.
+    ///
+    /// Activation dependency: when the blacklist reassigns a slot, the replacement also
+    /// signs the chunk's state witness. The witness-validation path must resolve the
+    /// producer via `prev_block_hash` against this same `DBCol::ChunkProducers` row (V2
+    /// partial witness, PR #15640) rather than the V1 `ChunkProductionKey` lookup, which
+    /// ignores the blacklist. `EarlyKickout` must not be activated before #15640 lands.
     pub fn save_chunk_producers_for_header(
         &mut self,
         epoch_manager: &dyn EpochManagerAdapter,
@@ -1846,52 +1855,50 @@ impl<'a> ChainStoreUpdate<'a> {
         if !ProtocolFeature::EarlyKickout.enabled(protocol_version) {
             return Ok(());
         }
-        let prev_block_hash = header.hash();
-        let epoch_id = epoch_manager.get_epoch_id_from_prev_block(prev_block_hash)?;
+        let anchor_hash = header.hash();
+        let epoch_id = epoch_manager.get_epoch_id_from_prev_block(anchor_hash)?;
         let shard_layout = epoch_manager.get_shard_layout(&epoch_id)?;
         let epoch_info = epoch_manager.get_epoch_info(&epoch_id)?;
-        let height = header.height() + 1;
+        let height = header.height() + CHUNK_GRANDPARENT_ANCHOR_HEIGHT_OFFSET;
 
-        for shard_id in shard_layout.shard_ids() {
-            if let Some(validator_id) =
-                epoch_info.sample_chunk_producer(&shard_layout, shard_id, height)
-            {
-                let validator_stake = epoch_info.get_validator(validator_id);
-                self.chain_store_cache_update
-                    .chunk_producers
-                    .insert((*prev_block_hash, shard_id), validator_stake);
-            }
-        }
-        Ok(())
-    }
-
-    /// Save chunk producers for genesis chunks which have
-    /// prev_block_hash = CryptoHash::default(). This is called once during
-    /// genesis init so that get_chunk_producer_info_db works for genesis chunks.
-    ///
-    /// Gated behind `EarlyKickout` protocol feature. No-op when disabled.
-    pub fn save_genesis_chunk_producers(
-        &mut self,
-        epoch_manager: &dyn EpochManagerAdapter,
-        protocol_version: ProtocolVersion,
-        genesis_height: BlockHeight,
-    ) -> Result<(), Error> {
-        if !ProtocolFeature::EarlyKickout.enabled(protocol_version) {
-            return Ok(());
-        }
-        let default_hash = CryptoHash::default();
-        let epoch_id = epoch_manager.get_epoch_id_from_prev_block(&default_hash)?;
-        let shard_layout = epoch_manager.get_shard_layout(&epoch_id)?;
-        let epoch_info = epoch_manager.get_epoch_info(&epoch_id)?;
+        // The blacklist excludes chunk producers that have been kicked out
+        // mid-epoch; excluded slots reassign to other producers on the same shard.
+        // Off-feature the blacklist is empty and sampling matches `sample_chunk_producer`.
+        let blacklist = epoch_manager.get_chunk_producer_blacklist(anchor_hash)?;
+        let empty = HashSet::new();
 
         for shard_id in shard_layout.shard_ids() {
-            if let Some(validator_id) =
-                epoch_info.sample_chunk_producer(&shard_layout, shard_id, genesis_height)
-            {
+            let shard_blacklist = blacklist.get(&shard_id).unwrap_or(&empty);
+            if let Some(validator_id) = epoch_info.sample_chunk_producer_excluding(
+                &shard_layout,
+                shard_id,
+                height,
+                shard_blacklist,
+            ) {
+                // Log a slot reassignment only when the default (un-blacklisted) pick is
+                // itself blacklisted and the exclusion changed the selected producer.
+                // Weighted samplers renormalize on any exclusion, so a differing pick
+                // alone does not imply the default producer was kicked.
+                if !shard_blacklist.is_empty() {
+                    if let Some(default_id) =
+                        epoch_info.sample_chunk_producer(&shard_layout, shard_id, height)
+                    {
+                        if shard_blacklist.contains(&default_id) && default_id != validator_id {
+                            tracing::info!(
+                                target: "early_kickout",
+                                %shard_id,
+                                height,
+                                kicked = %epoch_info.validator_account_id(default_id),
+                                reassigned_to = %epoch_info.validator_account_id(validator_id),
+                                "chunk producer slot reassigned"
+                            );
+                        }
+                    }
+                }
                 let validator_stake = epoch_info.get_validator(validator_id);
                 self.chain_store_cache_update
                     .chunk_producers
-                    .insert((default_hash, shard_id), validator_stake);
+                    .insert((*anchor_hash, shard_id), validator_stake);
             }
         }
         Ok(())
@@ -2130,7 +2137,6 @@ impl<'a> ChainStoreUpdate<'a> {
             }
         }
 
-        #[cfg(feature = "nightly")]
         for ((block_hash, shard_id), validator_stake) in
             &self.chain_store_cache_update.chunk_producers
         {