From d08a9829ad2049d6163dcd7511476384c41eca9a Mon Sep 17 00:00:00 2001 From: wacban Date: Fri, 19 Jun 2026 13:11:40 +0000 Subject: [PATCH 01/14] feat(rpc): report why a tx-status request timed out Transaction-status requests (`tx`, `EXPERIMENTAL_tx_status`, and the `wait_until` path of `send_tx`/`broadcast_tx_commit`) that time out before reaching the requested `wait_until` finality previously returned a context-free `TIMEOUT_ERROR`. They now return a `TIMEOUT_ERROR` whose `reason` explains what happened: - `pending`: the transaction was observed but is still below the requested finality; carries the last-known status so callers can re-poll. - `not_observed`: the transaction was never seen on chain. - `error`: the node could not produce a status before the timeout. Also refactors `tx_status_fetch` into focused helpers (`poll_tx_status`, `detect_invalid_tx`, `tx_status_on_timeout`). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 1 + .../src/types/transactions.rs | 84 ++++++++- chain/jsonrpc/src/api/transactions.rs | 9 +- chain/jsonrpc/src/lib.rs | 172 ++++++++++++------ 4 files changed, 206 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a031e710dc..9ff0a78605c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Protocol Changes ### Non-protocol Changes +* Transaction-status requests (`tx`, `EXPERIMENTAL_tx_status`, and the `wait_until` path of `send_tx`/`broadcast_tx_commit`) that time out before reaching the requested `wait_until` finality now return a `TIMEOUT_ERROR` carrying a `reason` that explains what happened: `pending` (includes the last-known transaction status so callers can see how far it got and re-poll for a higher finality), `not_observed` (the transaction was never seen on chain), or `error` (the node could not produce a status, with `debug_info`). Previously the timeout was context-free. ## [2.13.0] diff --git a/chain/jsonrpc-primitives/src/types/transactions.rs b/chain/jsonrpc-primitives/src/types/transactions.rs index 6b71c87ac8e..90d84eedeaf 100644 --- a/chain/jsonrpc-primitives/src/types/transactions.rs +++ b/chain/jsonrpc-primitives/src/types/transactions.rs @@ -54,10 +54,13 @@ pub enum RpcTransactionError { #[error("The node reached its limits. Try again later. More details: {debug_info}")] InternalError { debug_info: String }, #[error("Timeout")] - TimeoutError, + TimeoutError { + /// Why the request timed out before reaching the requested `wait_until` finality. + reason: RpcTransactionTimeoutReason, + }, } -#[derive(serde::Serialize, serde::Deserialize, Debug)] +#[derive(Clone, serde::Serialize, serde::Deserialize, Debug)] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct RpcTransactionResponse { #[serde(flatten)] @@ -65,6 +68,23 @@ pub struct RpcTransactionResponse { pub final_execution_status: near_primitives::views::TxExecutionStatus, } +/// Explains why a transaction-status request returned a `RpcTransactionError::TimeoutError`: +/// it did not reach the requested `wait_until` finality within the node's polling timeout. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum RpcTransactionTimeoutReason { + /// The node never observed the transaction on chain. + NotObserved, + /// The transaction was observed but is still pending the requested finality. The + /// last-known status is included so the caller can re-poll for a higher finality. + /// Boxed to keep `RpcTransactionError` small (it is the `Err` type of many RPC results). + Pending { status: Box }, + /// The node could not produce a usable transaction status before the timeout (for + /// example a repeated internal error, or no response at all). + Error { debug_info: String }, +} + #[derive(serde::Serialize, serde::Deserialize, Debug)] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct RpcBroadcastTxSyncResponse { @@ -140,3 +160,63 @@ impl From for crate::errors::RpcError { Self::new_internal_or_handler_error(Some(error_data), error_data_value) } } + +#[cfg(test)] +mod tests { + use super::*; + use near_primitives::views::TxExecutionStatus; + + /// On timeout the RPC returns a `TimeoutError` whose `reason` says how far the + /// transaction got. The `Pending` reason carries the last-known status so callers + /// retain full information and can re-poll for a higher finality. + #[test] + fn timeout_error_reports_pending() { + let error = RpcTransactionError::TimeoutError { + reason: RpcTransactionTimeoutReason::Pending { + status: Box::new(RpcTransactionResponse { + final_execution_outcome: None, + final_execution_status: TxExecutionStatus::Included, + }), + }, + }; + + // The reason and last-known status survive the conversion to the wire `RpcError`. + let rpc_error: crate::errors::RpcError = error.into(); + let wire = serde_json::to_value(&rpc_error).unwrap(); + assert_eq!(wire["cause"]["name"], "TIMEOUT_ERROR"); + let reason = &wire["cause"]["info"]["reason"]; + assert_eq!(reason["pending"]["status"]["final_execution_status"], "INCLUDED"); + } + + /// The `NotObserved` reason serializes as a bare tag with no payload. + #[test] + fn timeout_error_reports_not_observed() { + let error = + RpcTransactionError::TimeoutError { reason: RpcTransactionTimeoutReason::NotObserved }; + let value = serde_json::to_value(&error).unwrap(); + assert_eq!(value["name"], "TIMEOUT_ERROR"); + assert_eq!(value["info"]["reason"], "not_observed"); + } + + /// The error round-trips through serde, including the flattened status carried by + /// `Pending`. + #[test] + fn timeout_error_round_trips() { + let error = RpcTransactionError::TimeoutError { + reason: RpcTransactionTimeoutReason::Pending { + status: Box::new(RpcTransactionResponse { + final_execution_outcome: None, + final_execution_status: TxExecutionStatus::Included, + }), + }, + }; + let json = serde_json::to_string(&error).unwrap(); + let decoded: RpcTransactionError = serde_json::from_str(&json).unwrap(); + assert!(matches!( + decoded, + RpcTransactionError::TimeoutError { + reason: RpcTransactionTimeoutReason::Pending { .. } + } + )); + } +} diff --git a/chain/jsonrpc/src/api/transactions.rs b/chain/jsonrpc/src/api/transactions.rs index 943c55e1182..780c096c3e9 100644 --- a/chain/jsonrpc/src/api/transactions.rs +++ b/chain/jsonrpc/src/api/transactions.rs @@ -3,7 +3,8 @@ use near_async::messaging::AsyncSendError; use near_client_primitives::types::TxStatusError; use near_jsonrpc_primitives::errors::RpcParseError; use near_jsonrpc_primitives::types::transactions::{ - RpcSendTransactionRequest, RpcTransactionError, RpcTransactionStatusRequest, TransactionInfo, + RpcSendTransactionRequest, RpcTransactionError, RpcTransactionStatusRequest, + RpcTransactionTimeoutReason, TransactionInfo, }; use near_primitives::borsh::BorshDeserialize; use near_primitives::transaction::SignedTransaction; @@ -67,7 +68,11 @@ impl RpcFrom for RpcTransactionError { Self::UnknownTransaction { requested_transaction_hash } } TxStatusError::InternalError(debug_info) => Self::InternalError { debug_info }, - TxStatusError::TimeoutError => Self::TimeoutError, + TxStatusError::TimeoutError => Self::TimeoutError { + reason: RpcTransactionTimeoutReason::Error { + debug_info: "the node timed out fetching the transaction status".to_string(), + }, + }, } } } diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index 53f70128137..1829e112480 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -58,6 +58,7 @@ use near_jsonrpc_primitives::types::split_storage::{ }; use near_jsonrpc_primitives::types::transactions::{ RpcSendTransactionRequest, RpcTransactionError, RpcTransactionResponse, + RpcTransactionTimeoutReason, TransactionInfo, }; use near_jsonrpc_primitives::types::view_access_key::{ RpcViewAccessKeyError, RpcViewAccessKeyRequest, RpcViewAccessKeyResponse, @@ -81,6 +82,7 @@ use near_network::debug::GetDebugStatus; use near_network::tcp::{self, ListenerAddr}; use near_o11y::metrics::{Encoder, TextEncoder, prometheus}; use near_o11y::span_wrapped_msg::{SpanWrapped, SpanWrappedMessageExt}; +use near_primitives::errors::InvalidTxError; use near_primitives::hash::CryptoHash; use near_primitives::shard_layout::{ShardLayout, ShardUId}; use near_primitives::sharding::ChunkHash; @@ -104,6 +106,7 @@ use sharded_rpc::{ use std::collections::HashSet; use std::future::Future; use std::net::SocketAddr; +use std::ops::ControlFlow; use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; @@ -347,7 +350,11 @@ impl near_jsonrpc_primitives::types::transactions::RpcTransactionError { pub fn from_network_client_responses(resp: ProcessTxResponse) -> Self { match resp { ProcessTxResponse::InvalidTx(context) => Self::InvalidTransaction { context }, - ProcessTxResponse::NoResponse => Self::TimeoutError, + ProcessTxResponse::NoResponse => Self::TimeoutError { + reason: RpcTransactionTimeoutReason::Error { + debug_info: "no response from the node".to_string(), + }, + }, ProcessTxResponse::DoesNotTrackShard | ProcessTxResponse::RequestRouted => { Self::DoesNotTrackShard } @@ -915,7 +922,9 @@ impl JsonRpcHandler { ?signer_account_id, "timeout: tx_exists method" ); - near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError + near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError { + reason: RpcTransactionTimeoutReason::NotObserved, + } })? } @@ -931,64 +940,115 @@ impl JsonRpcHandler { near_jsonrpc_primitives::types::transactions::RpcTransactionResponse, near_jsonrpc_primitives::types::transactions::RpcTransactionError, > { - let (tx_hash, account_id) = tx_info.to_tx_hash_and_account(); - let mut tx_status_result = - Err(near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError); - self.clock.timeout(self.polling_config.polling_timeout, async { - // Create a new watch::Receiver to watch for new blocks. Mark the current block as seen. - let mut new_block_watcher = self.block_notification_watcher.clone(); - new_block_watcher.mark_unchanged(); + // If the request times out before any poll completes we report that we never got a + // usable status; each poll that keeps us waiting replaces this with a better reason. + let mut timeout_reason = RpcTransactionTimeoutReason::Error { + debug_info: "the node did not return a transaction status before the request timed out" + .to_string(), + }; - loop { - tx_status_result = self.view_client_send( TxStatus { - tx_hash, - signer_account_id: account_id.clone(), - fetch_receipt, - }) - .await; - match tx_status_result.clone() { - Ok(result) => { - if tx_execution_status_meets_expectations(&finality, &result.status) { - break Ok(result.into()) - } - // else: No such transaction recorded on chain yet - }, - Err(err @ near_jsonrpc_primitives::types::transactions::RpcTransactionError::UnknownTransaction { - .. - }) => { - if let Some(tx) = tx_info.to_signed_tx() { - if let Ok(ProcessTxResponse::InvalidTx(context)) = - self.send_tx_internal(tx.clone(), true).await - { - break Err( - near_jsonrpc_primitives::types::transactions::RpcTransactionError::InvalidTransaction { - context - } - ); - } - } - if finality == TxExecutionStatus::None { - break Err(err); - } + self.clock + .timeout(self.polling_config.polling_timeout, async { + // Create a new watch::Receiver to watch for new blocks. Mark the current block as seen. + let mut new_block_watcher = self.block_notification_watcher.clone(); + new_block_watcher.mark_unchanged(); + + loop { + match self.poll_tx_status(&tx_info, &finality, fetch_receipt).await { + ControlFlow::Break(outcome) => break outcome, + // Remember why we'd time out; reported if we actually do. + ControlFlow::Continue(reason) => timeout_reason = reason, } - Err(err) => break Err(err), + new_block_watcher.changed().await.map_err(|_| { + RpcTransactionError::InternalError { + debug_info: "block notification channel closed".to_string(), + } + })?; + } + }) + .await + // The polling loop returns on its own once it reaches the requested finality or + // hits a definitive error; only a timeout falls through to `unwrap_or_else`. + .unwrap_or_else(|_| self.tx_status_on_timeout(&tx_info, fetch_receipt, timeout_reason)) + } + + /// Performs a single `TxStatus` poll for `tx_status_fetch`. Returns `ControlFlow::Break` + /// with the final response/error once the transaction reaches the requested `finality` + /// or hits a definitive error, or `ControlFlow::Continue` with the reason to report if + /// the request ultimately times out. + async fn poll_tx_status( + &self, + tx_info: &TransactionInfo, + finality: &TxExecutionStatus, + fetch_receipt: bool, + ) -> ControlFlow, RpcTransactionTimeoutReason> + { + let (tx_hash, account_id) = tx_info.to_tx_hash_and_account(); + let request = TxStatus { tx_hash, signer_account_id: account_id.clone(), fetch_receipt }; + match self.view_client_send(request).await { + Ok(result) => { + // Stop as soon as we reach the requested finality; otherwise the transaction + // is on its way but not there yet, so keep polling, remembering how far it got. + if tx_execution_status_meets_expectations(finality, &result.status) { + ControlFlow::Break(Ok(result.into())) + } else { + ControlFlow::Continue(RpcTransactionTimeoutReason::Pending { + status: Box::new(result.into()), + }) } - new_block_watcher.changed().await.map_err(|_| RpcTransactionError::InternalError { debug_info: "Block notification channel closed".to_string() })?; } - }) - .await - .map_err(|_| { - metrics::RPC_TIMEOUT_TOTAL.inc(); - tracing::warn!( - target: "jsonrpc", - ?tx_info, - ?fetch_receipt, - ?tx_status_result, - timeout = ?self.polling_config.polling_timeout, - "timeout: tx_status_fetch method" - ); - near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError - })? + // Not recorded on chain (yet). If we were handed the signed transaction we can + // detect an invalid one right away; otherwise only `wait_until: NONE` asks us to + // stop now instead of waiting for it to appear. + Err(err @ RpcTransactionError::UnknownTransaction { .. }) => { + if let Some(context) = self.detect_invalid_tx(tx_info).await { + let error = RpcTransactionError::InvalidTransaction { context }; + return ControlFlow::Break(Err(error)); + } + if *finality == TxExecutionStatus::None { + ControlFlow::Break(Err(err)) + } else { + ControlFlow::Continue(RpcTransactionTimeoutReason::NotObserved) + } + } + // Any other error is terminal; surface it directly rather than waiting. + Err(err) => ControlFlow::Break(Err(err)), + } + } + + /// Detects a transaction that is invalid — and therefore will never appear on chain — + /// so the caller can fail fast instead of polling for it until the request times out. + /// This is only possible when we were handed the full signed transaction rather than + /// just its hash; it re-validates the transaction and returns the validation error if + /// it is invalid, or `None` if the transaction is valid or we only have its hash. + async fn detect_invalid_tx(&self, tx_info: &TransactionInfo) -> Option { + let tx = tx_info.to_signed_tx()?; + match self.send_tx_internal(tx.clone(), true).await { + Ok(ProcessTxResponse::InvalidTx(context)) => Some(context), + _ => None, + } + } + + /// Builds the `TimeoutError` returned when `tx_status_fetch`'s polling loop is cancelled + /// by the timeout. The `reason` records why the request did not reach the requested + /// finality in time (see `RpcTransactionTimeoutReason`), so the caller still has full + /// information and can re-poll. + fn tx_status_on_timeout( + &self, + tx_info: &TransactionInfo, + fetch_receipt: bool, + reason: RpcTransactionTimeoutReason, + ) -> Result { + metrics::RPC_TIMEOUT_TOTAL.inc(); + tracing::warn!( + target: "jsonrpc", + ?tx_info, + ?fetch_receipt, + ?reason, + timeout = ?self.polling_config.polling_timeout, + "timeout: tx_status_fetch method" + ); + Err(RpcTransactionError::TimeoutError { reason }) } /// Send a transaction idempotently (subsequent send of the same transaction will not cause From ade6bbb4a3fb7ac0a50127dedeec4004faf41ba2 Mon Sep 17 00:00:00 2001 From: wacban Date: Fri, 19 Jun 2026 15:47:49 +0200 Subject: [PATCH 02/14] nits --- CHANGELOG.md | 2 +- .../src/types/transactions.rs | 18 +++----- chain/jsonrpc/src/api/transactions.rs | 4 +- chain/jsonrpc/src/lib.rs | 45 +++++++++---------- 4 files changed, 31 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ff0a78605c..c8d582e0a2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Protocol Changes ### Non-protocol Changes -* Transaction-status requests (`tx`, `EXPERIMENTAL_tx_status`, and the `wait_until` path of `send_tx`/`broadcast_tx_commit`) that time out before reaching the requested `wait_until` finality now return a `TIMEOUT_ERROR` carrying a `reason` that explains what happened: `pending` (includes the last-known transaction status so callers can see how far it got and re-poll for a higher finality), `not_observed` (the transaction was never seen on chain), or `error` (the node could not produce a status, with `debug_info`). Previously the timeout was context-free. +* Transaction-status timeouts (`tx`, `EXPERIMENTAL_tx_status`, and `send_tx`/`broadcast_tx_commit` with `wait_until`) now carry a `reason` in the `TIMEOUT_ERROR`: `pending` (with the last-known status), `not_observed`, or `error`. Previously the timeout gave no detail. ## [2.13.0] diff --git a/chain/jsonrpc-primitives/src/types/transactions.rs b/chain/jsonrpc-primitives/src/types/transactions.rs index 90d84eedeaf..74bbc173523 100644 --- a/chain/jsonrpc-primitives/src/types/transactions.rs +++ b/chain/jsonrpc-primitives/src/types/transactions.rs @@ -54,10 +54,7 @@ pub enum RpcTransactionError { #[error("The node reached its limits. Try again later. More details: {debug_info}")] InternalError { debug_info: String }, #[error("Timeout")] - TimeoutError { - /// Why the request timed out before reaching the requested `wait_until` finality. - reason: RpcTransactionTimeoutReason, - }, + TimeoutError { reason: TimeoutErrorReason }, } #[derive(Clone, serde::Serialize, serde::Deserialize, Debug)] @@ -73,7 +70,7 @@ pub struct RpcTransactionResponse { #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] #[serde(rename_all = "snake_case")] -pub enum RpcTransactionTimeoutReason { +pub enum TimeoutErrorReason { /// The node never observed the transaction on chain. NotObserved, /// The transaction was observed but is still pending the requested finality. The @@ -172,7 +169,7 @@ mod tests { #[test] fn timeout_error_reports_pending() { let error = RpcTransactionError::TimeoutError { - reason: RpcTransactionTimeoutReason::Pending { + reason: TimeoutErrorReason::Pending { status: Box::new(RpcTransactionResponse { final_execution_outcome: None, final_execution_status: TxExecutionStatus::Included, @@ -191,8 +188,7 @@ mod tests { /// The `NotObserved` reason serializes as a bare tag with no payload. #[test] fn timeout_error_reports_not_observed() { - let error = - RpcTransactionError::TimeoutError { reason: RpcTransactionTimeoutReason::NotObserved }; + let error = RpcTransactionError::TimeoutError { reason: TimeoutErrorReason::NotObserved }; let value = serde_json::to_value(&error).unwrap(); assert_eq!(value["name"], "TIMEOUT_ERROR"); assert_eq!(value["info"]["reason"], "not_observed"); @@ -203,7 +199,7 @@ mod tests { #[test] fn timeout_error_round_trips() { let error = RpcTransactionError::TimeoutError { - reason: RpcTransactionTimeoutReason::Pending { + reason: TimeoutErrorReason::Pending { status: Box::new(RpcTransactionResponse { final_execution_outcome: None, final_execution_status: TxExecutionStatus::Included, @@ -214,9 +210,7 @@ mod tests { let decoded: RpcTransactionError = serde_json::from_str(&json).unwrap(); assert!(matches!( decoded, - RpcTransactionError::TimeoutError { - reason: RpcTransactionTimeoutReason::Pending { .. } - } + RpcTransactionError::TimeoutError { reason: TimeoutErrorReason::Pending { .. } } )); } } diff --git a/chain/jsonrpc/src/api/transactions.rs b/chain/jsonrpc/src/api/transactions.rs index 780c096c3e9..fa0bad1664c 100644 --- a/chain/jsonrpc/src/api/transactions.rs +++ b/chain/jsonrpc/src/api/transactions.rs @@ -4,7 +4,7 @@ use near_client_primitives::types::TxStatusError; use near_jsonrpc_primitives::errors::RpcParseError; use near_jsonrpc_primitives::types::transactions::{ RpcSendTransactionRequest, RpcTransactionError, RpcTransactionStatusRequest, - RpcTransactionTimeoutReason, TransactionInfo, + TimeoutErrorReason, TransactionInfo, }; use near_primitives::borsh::BorshDeserialize; use near_primitives::transaction::SignedTransaction; @@ -69,7 +69,7 @@ impl RpcFrom for RpcTransactionError { } TxStatusError::InternalError(debug_info) => Self::InternalError { debug_info }, TxStatusError::TimeoutError => Self::TimeoutError { - reason: RpcTransactionTimeoutReason::Error { + reason: TimeoutErrorReason::Error { debug_info: "the node timed out fetching the transaction status".to_string(), }, }, diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index 1829e112480..66b43330f53 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -57,8 +57,8 @@ use near_jsonrpc_primitives::types::split_storage::{ RpcSplitStorageInfoRequest, RpcSplitStorageInfoResponse, }; use near_jsonrpc_primitives::types::transactions::{ - RpcSendTransactionRequest, RpcTransactionError, RpcTransactionResponse, - RpcTransactionTimeoutReason, TransactionInfo, + RpcSendTransactionRequest, RpcTransactionError, RpcTransactionResponse, TimeoutErrorReason, + TransactionInfo, }; use near_jsonrpc_primitives::types::view_access_key::{ RpcViewAccessKeyError, RpcViewAccessKeyRequest, RpcViewAccessKeyResponse, @@ -351,7 +351,7 @@ impl near_jsonrpc_primitives::types::transactions::RpcTransactionError { match resp { ProcessTxResponse::InvalidTx(context) => Self::InvalidTransaction { context }, ProcessTxResponse::NoResponse => Self::TimeoutError { - reason: RpcTransactionTimeoutReason::Error { + reason: TimeoutErrorReason::Error { debug_info: "no response from the node".to_string(), }, }, @@ -923,7 +923,7 @@ impl JsonRpcHandler { "timeout: tx_exists method" ); near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError { - reason: RpcTransactionTimeoutReason::NotObserved, + reason: TimeoutErrorReason::NotObserved, } })? } @@ -942,7 +942,7 @@ impl JsonRpcHandler { > { // If the request times out before any poll completes we report that we never got a // usable status; each poll that keeps us waiting replaces this with a better reason. - let mut timeout_reason = RpcTransactionTimeoutReason::Error { + let mut timeout_reason = TimeoutErrorReason::Error { debug_info: "the node did not return a transaction status before the request timed out" .to_string(), }; @@ -981,8 +981,7 @@ impl JsonRpcHandler { tx_info: &TransactionInfo, finality: &TxExecutionStatus, fetch_receipt: bool, - ) -> ControlFlow, RpcTransactionTimeoutReason> - { + ) -> ControlFlow, TimeoutErrorReason> { let (tx_hash, account_id) = tx_info.to_tx_hash_and_account(); let request = TxStatus { tx_hash, signer_account_id: account_id.clone(), fetch_receipt }; match self.view_client_send(request).await { @@ -992,7 +991,7 @@ impl JsonRpcHandler { if tx_execution_status_meets_expectations(finality, &result.status) { ControlFlow::Break(Ok(result.into())) } else { - ControlFlow::Continue(RpcTransactionTimeoutReason::Pending { + ControlFlow::Continue(TimeoutErrorReason::Pending { status: Box::new(result.into()), }) } @@ -1001,14 +1000,14 @@ impl JsonRpcHandler { // detect an invalid one right away; otherwise only `wait_until: NONE` asks us to // stop now instead of waiting for it to appear. Err(err @ RpcTransactionError::UnknownTransaction { .. }) => { - if let Some(context) = self.detect_invalid_tx(tx_info).await { + if let Err(context) = self.validate_tx(tx_info).await { let error = RpcTransactionError::InvalidTransaction { context }; return ControlFlow::Break(Err(error)); } if *finality == TxExecutionStatus::None { ControlFlow::Break(Err(err)) } else { - ControlFlow::Continue(RpcTransactionTimeoutReason::NotObserved) + ControlFlow::Continue(TimeoutErrorReason::NotObserved) } } // Any other error is terminal; surface it directly rather than waiting. @@ -1016,28 +1015,28 @@ impl JsonRpcHandler { } } - /// Detects a transaction that is invalid — and therefore will never appear on chain — - /// so the caller can fail fast instead of polling for it until the request times out. - /// This is only possible when we were handed the full signed transaction rather than - /// just its hash; it re-validates the transaction and returns the validation error if - /// it is invalid, or `None` if the transaction is valid or we only have its hash. - async fn detect_invalid_tx(&self, tx_info: &TransactionInfo) -> Option { - let tx = tx_info.to_signed_tx()?; + /// Validates the transaction when we were handed the full signed transaction (rather + /// than just its hash), so the caller can fail fast on an invalid transaction instead + /// of polling for one that will never appear on chain. Returns `Err(context)` when the + /// transaction is known to be invalid, or `Ok(())` when it is valid or we cannot check + /// it (we only have its hash). + async fn validate_tx(&self, tx_info: &TransactionInfo) -> Result<(), InvalidTxError> { + let Some(tx) = tx_info.to_signed_tx() else { return Ok(()) }; match self.send_tx_internal(tx.clone(), true).await { - Ok(ProcessTxResponse::InvalidTx(context)) => Some(context), - _ => None, + Ok(ProcessTxResponse::InvalidTx(context)) => Err(context), + _ => Ok(()), } } /// Builds the `TimeoutError` returned when `tx_status_fetch`'s polling loop is cancelled /// by the timeout. The `reason` records why the request did not reach the requested - /// finality in time (see `RpcTransactionTimeoutReason`), so the caller still has full + /// finality in time (see `TimeoutErrorReason`), so the caller still has full /// information and can re-poll. fn tx_status_on_timeout( &self, tx_info: &TransactionInfo, fetch_receipt: bool, - reason: RpcTransactionTimeoutReason, + reason: TimeoutErrorReason, ) -> Result { metrics::RPC_TIMEOUT_TOTAL.inc(); tracing::warn!( @@ -2034,7 +2033,7 @@ impl JsonRpcHandler { // Coordinator-forwarded requests must only be answered for shards this // node actually applied. `block_effects` has no account filter, so the // scope is "every shard this node advertises tracking for at this - // epoch" — if any of those is missing a ChunkExtra, the response would + // epoch" - if any of those is missing a ChunkExtra, the response would // be silently partial, so we fail fast and let the coordinator retry. // Direct user requests preserve the legacy best-effort behavior. if source == RequestSource::Coordinator { @@ -2354,7 +2353,7 @@ impl JsonRpcHandler { error = %e, "scatter-gather: peer failed" ); - // Don't retry on request validation errors — they're + // Don't retry on request validation errors - they're // deterministic and every node will return the same. if matches!( e.error_struct.as_ref(), From d82899f7d42b915e03b52ab43f102afc7dbeb607 Mon Sep 17 00:00:00 2001 From: wacban Date: Fri, 19 Jun 2026 15:53:29 +0200 Subject: [PATCH 03/14] nit --- chain/jsonrpc/src/lib.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index 66b43330f53..a36add00ed2 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -956,7 +956,6 @@ impl JsonRpcHandler { loop { match self.poll_tx_status(&tx_info, &finality, fetch_receipt).await { ControlFlow::Break(outcome) => break outcome, - // Remember why we'd time out; reported if we actually do. ControlFlow::Continue(reason) => timeout_reason = reason, } new_block_watcher.changed().await.map_err(|_| { @@ -2033,7 +2032,7 @@ impl JsonRpcHandler { // Coordinator-forwarded requests must only be answered for shards this // node actually applied. `block_effects` has no account filter, so the // scope is "every shard this node advertises tracking for at this - // epoch" - if any of those is missing a ChunkExtra, the response would + // epoch" — if any of those is missing a ChunkExtra, the response would // be silently partial, so we fail fast and let the coordinator retry. // Direct user requests preserve the legacy best-effort behavior. if source == RequestSource::Coordinator { @@ -2353,7 +2352,7 @@ impl JsonRpcHandler { error = %e, "scatter-gather: peer failed" ); - // Don't retry on request validation errors - they're + // Don't retry on request validation errors — they're // deterministic and every node will return the same. if matches!( e.error_struct.as_ref(), From 4cf4d9ad0231427a73f5bee78aa07c1f4818bdaa Mon Sep 17 00:00:00 2001 From: wacban Date: Fri, 19 Jun 2026 14:05:29 +0000 Subject: [PATCH 04/14] chore(rpc): regenerate openapi spec for tx-status timeout reason Bump the spec version to 1.2.12 and regenerate it to include the new `TimeoutErrorReason` schema (`pending`/`not_observed`/`error`) on the `TIMEOUT_ERROR` of `RpcTransactionError`. Co-Authored-By: Claude Opus 4.8 (1M context) --- chain/jsonrpc/openapi/openapi.json | 70 +++++++++++++++++++++++++++++- chain/jsonrpc/openapi/src/main.rs | 2 +- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/chain/jsonrpc/openapi/openapi.json b/chain/jsonrpc/openapi/openapi.json index efb31134358..7fa322bd6ef 100644 --- a/chain/jsonrpc/openapi/openapi.json +++ b/chain/jsonrpc/openapi/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.0.0", "info": { "title": "NEAR Protocol JSON RPC API", - "version": "1.2.11" + "version": "1.2.12" }, "paths": { "/EXPERIMENTAL_call_function": { @@ -17877,6 +17877,17 @@ }, { "properties": { + "info": { + "properties": { + "reason": { + "$ref": "#/components/schemas/TimeoutErrorReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, "name": { "enum": [ "TIMEOUT_ERROR" @@ -17885,7 +17896,8 @@ } }, "required": [ - "name" + "name", + "info" ], "type": "object" } @@ -20799,6 +20811,60 @@ ], "type": "object" }, + "TimeoutErrorReason": { + "description": "Explains why a transaction-status request returned a `RpcTransactionError::TimeoutError`:\nit did not reach the requested `wait_until` finality within the node's polling timeout.", + "oneOf": [ + { + "description": "The node never observed the transaction on chain.", + "enum": [ + "not_observed" + ], + "type": "string" + }, + { + "additionalProperties": false, + "description": "The transaction was observed but is still pending the requested finality. The\nlast-known status is included so the caller can re-poll for a higher finality.\nBoxed to keep `RpcTransactionError` small (it is the `Err` type of many RPC results).", + "properties": { + "pending": { + "properties": { + "status": { + "$ref": "#/components/schemas/RpcTransactionResponse" + } + }, + "required": [ + "status" + ], + "type": "object" + } + }, + "required": [ + "pending" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The node could not produce a usable transaction status before the timeout (for\nexample a repeated internal error, or no response at all).", + "properties": { + "error": { + "properties": { + "debug_info": { + "type": "string" + } + }, + "required": [ + "debug_info" + ], + "type": "object" + } + }, + "required": [ + "error" + ], + "type": "object" + } + ] + }, "TrackedShardsConfig": { "description": "Describes the expected behavior of the node regarding shard tracking.\nIf the node is an active validator, it will also track the shards it is responsible for as a validator.", "oneOf": [ diff --git a/chain/jsonrpc/openapi/src/main.rs b/chain/jsonrpc/openapi/src/main.rs index d02ee95731f..4c010baea1d 100644 --- a/chain/jsonrpc/openapi/src/main.rs +++ b/chain/jsonrpc/openapi/src/main.rs @@ -643,7 +643,7 @@ fn whole_spec(all_schemas: SchemasMap, all_paths: PathsMap) -> OpenApi { openapi: "3.0.0".to_string(), info: okapi::openapi3::Info { title: "NEAR Protocol JSON RPC API".to_string(), - version: "1.2.11".to_string(), + version: "1.2.12".to_string(), ..Default::default() }, paths: all_paths, From 50e8c9889e7d54fb311fe76b20904edaaf91675b Mon Sep 17 00:00:00 2001 From: wacban Date: Mon, 22 Jun 2026 09:12:57 +0000 Subject: [PATCH 05/14] address copilot review --- chain/jsonrpc/src/lib.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index a36add00ed2..337a298746b 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -882,6 +882,7 @@ impl JsonRpcHandler { tx_hash: CryptoHash, signer_account_id: &AccountId, ) -> Result { + let mut last_error: Option = None; self.clock.timeout(self.polling_config.polling_timeout, async { // Create a new watch::Receiver to watch for new blocks. Mark the current block as seen. let mut new_block_watcher = self.block_notification_watcher.clone(); @@ -908,7 +909,7 @@ impl JsonRpcHandler { }) => { return Ok(false); } - _ => {} + Err(err) => last_error = Some(err), } new_block_watcher.changed().await.map_err(|_| RpcTransactionError::InternalError { debug_info: "Block notification channel closed".to_string() })?; } @@ -920,11 +921,12 @@ impl JsonRpcHandler { target: "jsonrpc", ?tx_hash, ?signer_account_id, + ?last_error, "timeout: tx_exists method" ); - near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError { - reason: TimeoutErrorReason::NotObserved, - } + let debug_info = format!("tx_exists timeout, last error: {:?}", last_error); + let reason = TimeoutErrorReason::Error { debug_info }; + near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError { reason } })? } From 3c44fba7ade409ee357a7b999c72bd618d682169 Mon Sep 17 00:00:00 2001 From: wacban Date: Fri, 26 Jun 2026 11:02:17 +0000 Subject: [PATCH 06/14] tag, cause & SCREAM --- .../src/types/transactions.rs | 49 +++++++------- chain/jsonrpc/openapi/openapi.json | 66 +++++++++---------- chain/jsonrpc/src/api/transactions.rs | 12 ++-- chain/jsonrpc/src/lib.rs | 38 +++++------ 4 files changed, 78 insertions(+), 87 deletions(-) diff --git a/chain/jsonrpc-primitives/src/types/transactions.rs b/chain/jsonrpc-primitives/src/types/transactions.rs index 74bbc173523..b8538543279 100644 --- a/chain/jsonrpc-primitives/src/types/transactions.rs +++ b/chain/jsonrpc-primitives/src/types/transactions.rs @@ -54,7 +54,7 @@ pub enum RpcTransactionError { #[error("The node reached its limits. Try again later. More details: {debug_info}")] InternalError { debug_info: String }, #[error("Timeout")] - TimeoutError { reason: TimeoutErrorReason }, + TimeoutError(TimeoutErrorCause), } #[derive(Clone, serde::Serialize, serde::Deserialize, Debug)] @@ -69,8 +69,8 @@ pub struct RpcTransactionResponse { /// it did not reach the requested `wait_until` finality within the node's polling timeout. #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum TimeoutErrorReason { +#[serde(tag = "cause", rename_all = "SCREAMING_SNAKE_CASE")] +pub enum TimeoutErrorCause { /// The node never observed the transaction on chain. NotObserved, /// The transaction was observed but is still pending the requested finality. The @@ -168,49 +168,46 @@ mod tests { /// retain full information and can re-poll for a higher finality. #[test] fn timeout_error_reports_pending() { - let error = RpcTransactionError::TimeoutError { - reason: TimeoutErrorReason::Pending { - status: Box::new(RpcTransactionResponse { - final_execution_outcome: None, - final_execution_status: TxExecutionStatus::Included, - }), - }, - }; - - // The reason and last-known status survive the conversion to the wire `RpcError`. + let error = RpcTransactionError::TimeoutError(TimeoutErrorCause::Pending { + status: Box::new(RpcTransactionResponse { + final_execution_outcome: None, + final_execution_status: TxExecutionStatus::Included, + }), + }); + + // The cause and last-known status survive the conversion to the wire `RpcError`. let rpc_error: crate::errors::RpcError = error.into(); let wire = serde_json::to_value(&rpc_error).unwrap(); assert_eq!(wire["cause"]["name"], "TIMEOUT_ERROR"); - let reason = &wire["cause"]["info"]["reason"]; - assert_eq!(reason["pending"]["status"]["final_execution_status"], "INCLUDED"); + let info = &wire["cause"]["info"]; + assert_eq!(info["cause"], "PENDING"); + assert_eq!(info["status"]["final_execution_status"], "INCLUDED"); } /// The `NotObserved` reason serializes as a bare tag with no payload. #[test] fn timeout_error_reports_not_observed() { - let error = RpcTransactionError::TimeoutError { reason: TimeoutErrorReason::NotObserved }; + let error = RpcTransactionError::TimeoutError(TimeoutErrorCause::NotObserved); let value = serde_json::to_value(&error).unwrap(); assert_eq!(value["name"], "TIMEOUT_ERROR"); - assert_eq!(value["info"]["reason"], "not_observed"); + assert_eq!(value["info"]["cause"], "NOT_OBSERVED"); } /// The error round-trips through serde, including the flattened status carried by /// `Pending`. #[test] fn timeout_error_round_trips() { - let error = RpcTransactionError::TimeoutError { - reason: TimeoutErrorReason::Pending { - status: Box::new(RpcTransactionResponse { - final_execution_outcome: None, - final_execution_status: TxExecutionStatus::Included, - }), - }, - }; + let error = RpcTransactionError::TimeoutError(TimeoutErrorCause::Pending { + status: Box::new(RpcTransactionResponse { + final_execution_outcome: None, + final_execution_status: TxExecutionStatus::Included, + }), + }); let json = serde_json::to_string(&error).unwrap(); let decoded: RpcTransactionError = serde_json::from_str(&json).unwrap(); assert!(matches!( decoded, - RpcTransactionError::TimeoutError { reason: TimeoutErrorReason::Pending { .. } } + RpcTransactionError::TimeoutError(TimeoutErrorCause::Pending { .. }) )); } } diff --git a/chain/jsonrpc/openapi/openapi.json b/chain/jsonrpc/openapi/openapi.json index 7fa322bd6ef..ca979259ae0 100644 --- a/chain/jsonrpc/openapi/openapi.json +++ b/chain/jsonrpc/openapi/openapi.json @@ -17878,15 +17878,7 @@ { "properties": { "info": { - "properties": { - "reason": { - "$ref": "#/components/schemas/TimeoutErrorReason" - } - }, - "required": [ - "reason" - ], - "type": "object" + "$ref": "#/components/schemas/TimeoutErrorCause" }, "name": { "enum": [ @@ -20811,55 +20803,59 @@ ], "type": "object" }, - "TimeoutErrorReason": { + "TimeoutErrorCause": { "description": "Explains why a transaction-status request returned a `RpcTransactionError::TimeoutError`:\nit did not reach the requested `wait_until` finality within the node's polling timeout.", "oneOf": [ { "description": "The node never observed the transaction on chain.", - "enum": [ - "not_observed" + "properties": { + "cause": { + "enum": [ + "NOT_OBSERVED" + ], + "type": "string" + } + }, + "required": [ + "cause" ], - "type": "string" + "type": "object" }, { - "additionalProperties": false, "description": "The transaction was observed but is still pending the requested finality. The\nlast-known status is included so the caller can re-poll for a higher finality.\nBoxed to keep `RpcTransactionError` small (it is the `Err` type of many RPC results).", "properties": { - "pending": { - "properties": { - "status": { - "$ref": "#/components/schemas/RpcTransactionResponse" - } - }, - "required": [ - "status" + "cause": { + "enum": [ + "PENDING" ], - "type": "object" + "type": "string" + }, + "status": { + "$ref": "#/components/schemas/RpcTransactionResponse" } }, "required": [ - "pending" + "cause", + "status" ], "type": "object" }, { - "additionalProperties": false, "description": "The node could not produce a usable transaction status before the timeout (for\nexample a repeated internal error, or no response at all).", "properties": { - "error": { - "properties": { - "debug_info": { - "type": "string" - } - }, - "required": [ - "debug_info" + "cause": { + "enum": [ + "ERROR" ], - "type": "object" + "type": "string" + }, + "debug_info": { + "type": "string" } }, "required": [ - "error" + "cause", + "debug_info" ], "type": "object" } diff --git a/chain/jsonrpc/src/api/transactions.rs b/chain/jsonrpc/src/api/transactions.rs index fa0bad1664c..8ed44c2d443 100644 --- a/chain/jsonrpc/src/api/transactions.rs +++ b/chain/jsonrpc/src/api/transactions.rs @@ -3,8 +3,8 @@ use near_async::messaging::AsyncSendError; use near_client_primitives::types::TxStatusError; use near_jsonrpc_primitives::errors::RpcParseError; use near_jsonrpc_primitives::types::transactions::{ - RpcSendTransactionRequest, RpcTransactionError, RpcTransactionStatusRequest, - TimeoutErrorReason, TransactionInfo, + RpcSendTransactionRequest, RpcTransactionError, RpcTransactionStatusRequest, TimeoutErrorCause, + TransactionInfo, }; use near_primitives::borsh::BorshDeserialize; use near_primitives::transaction::SignedTransaction; @@ -68,11 +68,9 @@ impl RpcFrom for RpcTransactionError { Self::UnknownTransaction { requested_transaction_hash } } TxStatusError::InternalError(debug_info) => Self::InternalError { debug_info }, - TxStatusError::TimeoutError => Self::TimeoutError { - reason: TimeoutErrorReason::Error { - debug_info: "the node timed out fetching the transaction status".to_string(), - }, - }, + TxStatusError::TimeoutError => Self::TimeoutError(TimeoutErrorCause::Error { + debug_info: "the node timed out fetching the transaction status".to_string(), + }), } } } diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index 337a298746b..1a93b4631da 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -57,7 +57,7 @@ use near_jsonrpc_primitives::types::split_storage::{ RpcSplitStorageInfoRequest, RpcSplitStorageInfoResponse, }; use near_jsonrpc_primitives::types::transactions::{ - RpcSendTransactionRequest, RpcTransactionError, RpcTransactionResponse, TimeoutErrorReason, + RpcSendTransactionRequest, RpcTransactionError, RpcTransactionResponse, TimeoutErrorCause, TransactionInfo, }; use near_jsonrpc_primitives::types::view_access_key::{ @@ -350,11 +350,9 @@ impl near_jsonrpc_primitives::types::transactions::RpcTransactionError { pub fn from_network_client_responses(resp: ProcessTxResponse) -> Self { match resp { ProcessTxResponse::InvalidTx(context) => Self::InvalidTransaction { context }, - ProcessTxResponse::NoResponse => Self::TimeoutError { - reason: TimeoutErrorReason::Error { - debug_info: "no response from the node".to_string(), - }, - }, + ProcessTxResponse::NoResponse => Self::TimeoutError(TimeoutErrorCause::Error { + debug_info: "no response from the node".to_string(), + }), ProcessTxResponse::DoesNotTrackShard | ProcessTxResponse::RequestRouted => { Self::DoesNotTrackShard } @@ -925,8 +923,8 @@ impl JsonRpcHandler { "timeout: tx_exists method" ); let debug_info = format!("tx_exists timeout, last error: {:?}", last_error); - let reason = TimeoutErrorReason::Error { debug_info }; - near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError { reason } + let cause = TimeoutErrorCause::Error { debug_info }; + near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError(cause) })? } @@ -944,7 +942,7 @@ impl JsonRpcHandler { > { // If the request times out before any poll completes we report that we never got a // usable status; each poll that keeps us waiting replaces this with a better reason. - let mut timeout_reason = TimeoutErrorReason::Error { + let mut timeout_error_cause = TimeoutErrorCause::Error { debug_info: "the node did not return a transaction status before the request timed out" .to_string(), }; @@ -958,7 +956,7 @@ impl JsonRpcHandler { loop { match self.poll_tx_status(&tx_info, &finality, fetch_receipt).await { ControlFlow::Break(outcome) => break outcome, - ControlFlow::Continue(reason) => timeout_reason = reason, + ControlFlow::Continue(reason) => timeout_error_cause = reason, } new_block_watcher.changed().await.map_err(|_| { RpcTransactionError::InternalError { @@ -970,7 +968,9 @@ impl JsonRpcHandler { .await // The polling loop returns on its own once it reaches the requested finality or // hits a definitive error; only a timeout falls through to `unwrap_or_else`. - .unwrap_or_else(|_| self.tx_status_on_timeout(&tx_info, fetch_receipt, timeout_reason)) + .unwrap_or_else(|_| { + self.tx_status_on_timeout(&tx_info, fetch_receipt, timeout_error_cause) + }) } /// Performs a single `TxStatus` poll for `tx_status_fetch`. Returns `ControlFlow::Break` @@ -982,7 +982,7 @@ impl JsonRpcHandler { tx_info: &TransactionInfo, finality: &TxExecutionStatus, fetch_receipt: bool, - ) -> ControlFlow, TimeoutErrorReason> { + ) -> ControlFlow, TimeoutErrorCause> { let (tx_hash, account_id) = tx_info.to_tx_hash_and_account(); let request = TxStatus { tx_hash, signer_account_id: account_id.clone(), fetch_receipt }; match self.view_client_send(request).await { @@ -992,7 +992,7 @@ impl JsonRpcHandler { if tx_execution_status_meets_expectations(finality, &result.status) { ControlFlow::Break(Ok(result.into())) } else { - ControlFlow::Continue(TimeoutErrorReason::Pending { + ControlFlow::Continue(TimeoutErrorCause::Pending { status: Box::new(result.into()), }) } @@ -1008,7 +1008,7 @@ impl JsonRpcHandler { if *finality == TxExecutionStatus::None { ControlFlow::Break(Err(err)) } else { - ControlFlow::Continue(TimeoutErrorReason::NotObserved) + ControlFlow::Continue(TimeoutErrorCause::NotObserved) } } // Any other error is terminal; surface it directly rather than waiting. @@ -1030,25 +1030,25 @@ impl JsonRpcHandler { } /// Builds the `TimeoutError` returned when `tx_status_fetch`'s polling loop is cancelled - /// by the timeout. The `reason` records why the request did not reach the requested - /// finality in time (see `TimeoutErrorReason`), so the caller still has full + /// by the timeout. The `cause` records why the request did not reach the requested + /// finality in time (see `TimeoutErrorCause`), so the caller still has full /// information and can re-poll. fn tx_status_on_timeout( &self, tx_info: &TransactionInfo, fetch_receipt: bool, - reason: TimeoutErrorReason, + cause: TimeoutErrorCause, ) -> Result { metrics::RPC_TIMEOUT_TOTAL.inc(); tracing::warn!( target: "jsonrpc", ?tx_info, ?fetch_receipt, - ?reason, + ?cause, timeout = ?self.polling_config.polling_timeout, "timeout: tx_status_fetch method" ); - Err(RpcTransactionError::TimeoutError { reason }) + Err(RpcTransactionError::TimeoutError(cause)) } /// Send a transaction idempotently (subsequent send of the same transaction will not cause From 27f746ef6b55657b8c0ef87f22935493c61e1b94 Mon Sep 17 00:00:00 2001 From: wacban Date: Fri, 26 Jun 2026 12:07:42 +0000 Subject: [PATCH 07/14] TxStatusOutcome --- chain/client-primitives/src/types.rs | 15 +- chain/client/src/lib.rs | 2 +- chain/client/src/view_client_actor.rs | 53 +++---- .../src/types/transactions.rs | 5 +- chain/jsonrpc/openapi/openapi.json | 19 +++ chain/jsonrpc/src/lib.rs | 131 ++++++++++-------- 6 files changed, 143 insertions(+), 82 deletions(-) diff --git a/chain/client-primitives/src/types.rs b/chain/client-primitives/src/types.rs index a2a37f6240e..0085ba4a8ca 100644 --- a/chain/client-primitives/src/types.rs +++ b/chain/client-primitives/src/types.rs @@ -9,7 +9,7 @@ use near_primitives::types::{ }; use near_primitives::views::{ EpochSyncStatusView, ExecutionOutcomeWithIdView, LightClientBlockLiteView, QueryRequest, - StateChangesRequestView, StateSyncStatusView, SyncStatusView, + StateChangesRequestView, StateSyncStatusView, SyncStatusView, TxStatusView, }; pub use near_primitives::views::{StatusResponse, StatusSyncInfo}; use near_time::Duration; @@ -626,6 +626,19 @@ pub struct TxStatus { pub fetch_receipt: bool, } +/// Outcome of the transaction status lookup, including either the status or +/// full context on why the status is unavailable. +#[derive(Debug)] +pub enum TxStatusOutcome { + /// The node tracks the transaction's shard and observed it. + Observed(TxStatusView), + /// The node tracks the shard but has not seen the transaction on chain. + NotObserved, + /// The node does not track the transaction's shard; the query was forwarded + /// to a chunk producer that does, and no answer is available yet. + ShardNotTracked { shard_id: ShardId }, +} + #[derive(Debug)] pub enum TxStatusError { ChainError(near_chain_primitives::Error), diff --git a/chain/client/src/lib.rs b/chain/client/src/lib.rs index b2ff45c015b..5f165946294 100644 --- a/chain/client/src/lib.rs +++ b/chain/client/src/lib.rs @@ -26,7 +26,7 @@ pub use near_client_primitives::types::{ GetReceiptToTxResponse, GetShardChunk, GetSplitStorageInfo, GetStateChanges, GetStateChangesInBlock, GetStateChangesWithCauseInBlock, GetStateChangesWithCauseInBlockForTrackedShards, GetValidatorInfo, GetValidatorOrdered, Query, - QueryError, Status, StatusResponse, SyncStatus, TxStatus, TxStatusError, + QueryError, Status, StatusResponse, SyncStatus, TxStatus, TxStatusError, TxStatusOutcome, }; pub use near_network::client::{ BlockApproval, BlockResponse, ProcessTxRequest, ProcessTxResponse, SetNetworkInfo, diff --git a/chain/client/src/view_client_actor.rs b/chain/client/src/view_client_actor.rs index ccad4978558..2a906ae3ced 100644 --- a/chain/client/src/view_client_actor.rs +++ b/chain/client/src/view_client_actor.rs @@ -28,7 +28,7 @@ use near_client_primitives::types::{ GetReceipt, GetReceiptError, GetReceiptToTx, GetReceiptToTxError, GetReceiptToTxResponse, GetSplitStorageInfo, GetSplitStorageInfoError, GetStateChangesError, GetStateChangesWithCauseInBlock, GetStateChangesWithCauseInBlockForTrackedShards, - GetValidatorInfoError, Query, QueryError, TxStatus, TxStatusError, + GetValidatorInfoError, Query, QueryError, TxStatus, TxStatusError, TxStatusOutcome, }; use near_epoch_manager::EpochManagerAdapter; use near_epoch_manager::shard_assignment::{account_id_to_shard_id, shard_id_to_uid}; @@ -637,7 +637,7 @@ impl ViewClientActor { tx_hash: CryptoHash, signer_account_id: AccountId, fetch_receipt: bool, - ) -> Result { + ) -> Result { { // TODO(telezhnaya): take into account `fetch_receipt()` // https://github.com/near/nearcore/issues/9545 @@ -645,12 +645,9 @@ impl ViewClientActor { if let Some(res) = request_manager.tx_status_response.pop(&tx_hash) { request_manager.tx_status_requests.pop(&tx_hash); let status = self.get_tx_execution_status(&res)?; - return Ok(TxStatusView { - execution_outcome: Some(FinalExecutionOutcomeViewEnum::FinalExecutionOutcome( - res, - )), - status, - }); + let execution_outcome = + Some(FinalExecutionOutcomeViewEnum::FinalExecutionOutcome(res)); + return Ok(TxStatusOutcome::Observed(TxStatusView { execution_outcome, status })); } } @@ -672,8 +669,14 @@ impl ViewClientActor { } else { FinalExecutionOutcomeViewEnum::FinalExecutionOutcome(tx_result) }; - Ok(TxStatusView { execution_outcome: Some(res), status }) + Ok(TxStatusOutcome::Observed(TxStatusView { + execution_outcome: Some(res), + status, + })) } + // TODO: `get_partial_transaction_result` returns `DBNotFoundErr` both when the + // tx is unknown and when its result is merely incomplete; have it return a + // partial result so we don't re-query the store and hardcode a coarse status. Err(near_chain::Error::DBNotFoundErr(_)) => { if let Some(transaction) = self.chain.chain_store.get_transaction(&tx_hash) { let transaction = @@ -687,18 +690,18 @@ impl ViewClientActor { receipts_outcome: vec![], }, ); - Ok(TxStatusView { + Ok(TxStatusOutcome::Observed(TxStatusView { execution_outcome: Some(outcome), status: TxExecutionStatus::Included, - }) + })) } else { - Ok(TxStatusView { + Ok(TxStatusOutcome::Observed(TxStatusView { execution_outcome: None, status: TxExecutionStatus::Included, - }) + })) } } else { - Err(TxStatusError::MissingTransaction(tx_hash)) + Ok(TxStatusOutcome::NotObserved) } } Err(err) => { @@ -729,7 +732,7 @@ impl ViewClientActor { NetworkRequests::TxStatus(validator, signer_account_id, tx_hash), )); } - Ok(TxStatusView { execution_outcome: None, status: TxExecutionStatus::None }) + Ok(TxStatusOutcome::ShardNotTracked { shard_id: target_shard_id }) } } @@ -876,8 +879,8 @@ impl Handler> for ViewClientActor { } } -impl Handler> for ViewClientActor { - fn handle(&mut self, msg: TxStatus) -> Result { +impl Handler> for ViewClientActor { + fn handle(&mut self, msg: TxStatus) -> Result { tracing::debug!(target: "client", ?msg); let _timer = metrics::VIEW_CLIENT_MESSAGE_TIME.with_label_values(&["TxStatus"]).start_timer(); @@ -1697,13 +1700,15 @@ impl Handler>> for ViewCl let _timer = metrics::VIEW_CLIENT_MESSAGE_TIME.with_label_values(&["TxStatusRequest"]).start_timer(); let TxStatusRequest { tx_hash, signer_account_id } = msg; - if let Ok(Some(result)) = - self.get_tx_status(tx_hash, signer_account_id, false).map(|s| s.execution_outcome) - { - Some(Box::new(result.into_outcome())) - } else { - None - } + let tx_status = self.get_tx_status(tx_hash, signer_account_id, false); + // TODO: the forwarded `TxStatusResponse` only carries the outcome, so the rest of + // `tx_status` (status, `NotObserved`) is dropped. Enrich the response so a forwarded + // query can relay a definitive "not observed" instead of going silent on the RPC node. + let outcome = match tx_status { + Ok(TxStatusOutcome::Observed(view)) => view.execution_outcome, + _ => None, + }; + outcome.map(|result| Box::new(result.into_outcome())) } } diff --git a/chain/jsonrpc-primitives/src/types/transactions.rs b/chain/jsonrpc-primitives/src/types/transactions.rs index b8538543279..6090628d9aa 100644 --- a/chain/jsonrpc-primitives/src/types/transactions.rs +++ b/chain/jsonrpc-primitives/src/types/transactions.rs @@ -1,5 +1,5 @@ use near_primitives::hash::CryptoHash; -use near_primitives::types::AccountId; +use near_primitives::types::{AccountId, ShardId}; use serde_json::Value; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -77,6 +77,9 @@ pub enum TimeoutErrorCause { /// last-known status is included so the caller can re-poll for a higher finality. /// Boxed to keep `RpcTransactionError` small (it is the `Err` type of many RPC results). Pending { status: Box }, + /// The node does not track the transaction's shard and could not get an answer from a + /// chunk producer that does before the timeout. + ShardNotTracked { shard_id: ShardId }, /// The node could not produce a usable transaction status before the timeout (for /// example a repeated internal error, or no response at all). Error { debug_info: String }, diff --git a/chain/jsonrpc/openapi/openapi.json b/chain/jsonrpc/openapi/openapi.json index ca979259ae0..9566c2268d8 100644 --- a/chain/jsonrpc/openapi/openapi.json +++ b/chain/jsonrpc/openapi/openapi.json @@ -20840,6 +20840,25 @@ ], "type": "object" }, + { + "description": "The node does not track the transaction's shard and could not get an answer from a\nchunk producer that does before the timeout.", + "properties": { + "cause": { + "enum": [ + "SHARD_NOT_TRACKED" + ], + "type": "string" + }, + "shard_id": { + "$ref": "#/components/schemas/ShardId" + } + }, + "required": [ + "cause", + "shard_id" + ], + "type": "object" + }, { "description": "The node could not produce a usable transaction status before the timeout (for\nexample a repeated internal error, or no response at all).", "properties": { diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index 1a93b4631da..0bb08ce8236 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -20,6 +20,7 @@ use near_client::{ GetReceiptToTx, GetReceiptToTxResponse, GetStateChanges, GetStateChangesInBlock, GetValidatorInfo, GetValidatorOrdered, ProcessTxRequest, ProcessTxResponse, Query as ClientQuery, QueryError, Status, StatusResponse, TxStatus, TxStatusError, + TxStatusOutcome, }; use near_client_primitives::debug::{ DebugBlockStatusQuery, DebugBlocksStartingMode, DebugStatusResponse, @@ -95,7 +96,7 @@ use near_primitives::views::{ BlockView, ChunkView, EpochValidatorInfo, GasPriceView, LightClientBlockView, MaintenanceWindowsView, QueryRequest, QueryResponse, ReceiptView, SplitStorageInfoView, StateChangeKindView, StateChangesKindsView, StateChangesRequestView, StateChangesView, - TxExecutionStatus, TxStatusView, + TxExecutionStatus, }; use parking_lot::RwLock; use serde_json::{Value, json}; @@ -468,7 +469,7 @@ pub struct ViewClientSenderForRpc( AsyncSender>, AsyncSender, GetValidatorInfoError>>, AsyncSender>, - AsyncSender>, + AsyncSender>, #[cfg(feature = "test_features")] Sender, ); @@ -881,51 +882,59 @@ impl JsonRpcHandler { signer_account_id: &AccountId, ) -> Result { let mut last_error: Option = None; - self.clock.timeout(self.polling_config.polling_timeout, async { - // Create a new watch::Receiver to watch for new blocks. Mark the current block as seen. - let mut new_block_watcher = self.block_notification_watcher.clone(); - new_block_watcher.mark_unchanged(); + self.clock + .timeout(self.polling_config.polling_timeout, async { + // Create a new watch::Receiver to watch for new blocks. Mark the current block as seen. + let mut new_block_watcher = self.block_notification_watcher.clone(); + new_block_watcher.mark_unchanged(); - loop { - // TODO(optimization): Introduce a view_client method to only get transaction - // status without the information about execution outcomes. - match self.view_client_send( - TxStatus { - tx_hash, - signer_account_id: signer_account_id.clone(), - fetch_receipt: false, - }) - .await - { - Ok(status) => { - if let Some(_) = status.execution_outcome { + loop { + // TODO(optimization): Introduce a view_client method to only get transaction + // status without the information about execution outcomes. + match self + .view_client_send(TxStatus { + tx_hash, + signer_account_id: signer_account_id.clone(), + fetch_receipt: false, + }) + .await + { + // Observed with an execution outcome: the transaction exists. + Ok(TxStatusOutcome::Observed(view)) if view.execution_outcome.is_some() => { return Ok(true); } + // Observed without an outcome yet, or shard not tracked (query forwarded): + // keep polling. + Ok( + TxStatusOutcome::Observed(_) | TxStatusOutcome::ShardNotTracked { .. }, + ) => {} + // Tracked shard, transaction not found: it does not exist. + Ok(TxStatusOutcome::NotObserved) => return Ok(false), + Err(err) => last_error = Some(err), } - Err(near_jsonrpc_primitives::types::transactions::RpcTransactionError::UnknownTransaction { - .. - }) => { - return Ok(false); - } - Err(err) => last_error = Some(err), + new_block_watcher.changed().await.map_err(|_| { + RpcTransactionError::InternalError { + debug_info: "Block notification channel closed".to_string(), + } + })?; } - new_block_watcher.changed().await.map_err(|_| RpcTransactionError::InternalError { debug_info: "Block notification channel closed".to_string() })?; - } - }) - .await - .map_err(|_| { - metrics::RPC_TIMEOUT_TOTAL.inc(); - tracing::warn!( - target: "jsonrpc", - ?tx_hash, - ?signer_account_id, - ?last_error, - "timeout: tx_exists method" - ); - let debug_info = format!("tx_exists timeout, last error: {:?}", last_error); - let cause = TimeoutErrorCause::Error { debug_info }; - near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError(cause) - })? + }) + .await + .map_err(|_| { + metrics::RPC_TIMEOUT_TOTAL.inc(); + tracing::warn!( + target: "jsonrpc", + ?tx_hash, + ?signer_account_id, + ?last_error, + "timeout: tx_exists method" + ); + let debug_info = format!("tx_exists timeout, last error: {:?}", last_error); + let cause = TimeoutErrorCause::Error { debug_info }; + near_jsonrpc_primitives::types::transactions::RpcTransactionError::TimeoutError( + cause, + ) + })? } /// Return status of the given transaction @@ -986,31 +995,43 @@ impl JsonRpcHandler { let (tx_hash, account_id) = tx_info.to_tx_hash_and_account(); let request = TxStatus { tx_hash, signer_account_id: account_id.clone(), fetch_receipt }; match self.view_client_send(request).await { - Ok(result) => { - // Stop as soon as we reach the requested finality; otherwise the transaction - // is on its way but not there yet, so keep polling, remembering how far it got. - if tx_execution_status_meets_expectations(finality, &result.status) { - ControlFlow::Break(Ok(result.into())) + // The node tracks the shard and observed the transaction. Stop once it reaches + // the requested finality; otherwise keep polling, remembering how far it got. + Ok(TxStatusOutcome::Observed(view)) => { + if tx_execution_status_meets_expectations(finality, &view.status) { + ControlFlow::Break(Ok(view.into())) } else { ControlFlow::Continue(TimeoutErrorCause::Pending { - status: Box::new(result.into()), + status: Box::new(view.into()), }) } } - // Not recorded on chain (yet). If we were handed the signed transaction we can - // detect an invalid one right away; otherwise only `wait_until: NONE` asks us to - // stop now instead of waiting for it to appear. - Err(err @ RpcTransactionError::UnknownTransaction { .. }) => { + // Tracked shard, transaction not on chain. Fail fast if we can prove it invalid; + // `wait_until: NONE` reports it unknown immediately, otherwise we keep waiting. + Ok(TxStatusOutcome::NotObserved) => { if let Err(context) = self.validate_tx(tx_info).await { - let error = RpcTransactionError::InvalidTransaction { context }; - return ControlFlow::Break(Err(error)); + return ControlFlow::Break(Err(RpcTransactionError::InvalidTransaction { + context, + })); } if *finality == TxExecutionStatus::None { - ControlFlow::Break(Err(err)) + ControlFlow::Break(Err(RpcTransactionError::UnknownTransaction { + requested_transaction_hash: tx_hash, + })) } else { ControlFlow::Continue(TimeoutErrorCause::NotObserved) } } + // We don't track the shard; the view client forwarded the query. `wait_until: + // NONE` says so immediately; otherwise wait for the forwarded answer and, if it + // never arrives, time out with the shard recorded. + Ok(TxStatusOutcome::ShardNotTracked { shard_id }) => { + if *finality == TxExecutionStatus::None { + ControlFlow::Break(Err(RpcTransactionError::DoesNotTrackShard)) + } else { + ControlFlow::Continue(TimeoutErrorCause::ShardNotTracked { shard_id }) + } + } // Any other error is terminal; surface it directly rather than waiting. Err(err) => ControlFlow::Break(Err(err)), } From c43d9181c6d8d0e192a282552c35215c481a4a8a Mon Sep 17 00:00:00 2001 From: wacban Date: Fri, 26 Jun 2026 12:35:20 +0000 Subject: [PATCH 08/14] nits --- chain/client-primitives/src/types.rs | 2 +- chain/client/src/view_client_actor.rs | 2 +- .../src/types/transactions.rs | 2 +- chain/jsonrpc/openapi/openapi.json | 2 +- chain/jsonrpc/src/lib.rs | 25 ++++++++----------- 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/chain/client-primitives/src/types.rs b/chain/client-primitives/src/types.rs index 0085ba4a8ca..e310cb2b13d 100644 --- a/chain/client-primitives/src/types.rs +++ b/chain/client-primitives/src/types.rs @@ -636,7 +636,7 @@ pub enum TxStatusOutcome { NotObserved, /// The node does not track the transaction's shard; the query was forwarded /// to a chunk producer that does, and no answer is available yet. - ShardNotTracked { shard_id: ShardId }, + DoesNotTrackShard { shard_id: ShardId }, } #[derive(Debug)] diff --git a/chain/client/src/view_client_actor.rs b/chain/client/src/view_client_actor.rs index 2a906ae3ced..0f7f8f77a5b 100644 --- a/chain/client/src/view_client_actor.rs +++ b/chain/client/src/view_client_actor.rs @@ -732,7 +732,7 @@ impl ViewClientActor { NetworkRequests::TxStatus(validator, signer_account_id, tx_hash), )); } - Ok(TxStatusOutcome::ShardNotTracked { shard_id: target_shard_id }) + Ok(TxStatusOutcome::DoesNotTrackShard { shard_id: target_shard_id }) } } diff --git a/chain/jsonrpc-primitives/src/types/transactions.rs b/chain/jsonrpc-primitives/src/types/transactions.rs index 6090628d9aa..464b9962dd0 100644 --- a/chain/jsonrpc-primitives/src/types/transactions.rs +++ b/chain/jsonrpc-primitives/src/types/transactions.rs @@ -79,7 +79,7 @@ pub enum TimeoutErrorCause { Pending { status: Box }, /// The node does not track the transaction's shard and could not get an answer from a /// chunk producer that does before the timeout. - ShardNotTracked { shard_id: ShardId }, + DoesNotTrackShard { shard_id: ShardId }, /// The node could not produce a usable transaction status before the timeout (for /// example a repeated internal error, or no response at all). Error { debug_info: String }, diff --git a/chain/jsonrpc/openapi/openapi.json b/chain/jsonrpc/openapi/openapi.json index 9566c2268d8..04ea5e3a820 100644 --- a/chain/jsonrpc/openapi/openapi.json +++ b/chain/jsonrpc/openapi/openapi.json @@ -20845,7 +20845,7 @@ "properties": { "cause": { "enum": [ - "SHARD_NOT_TRACKED" + "DOES_NOT_TRACK_SHARD" ], "type": "string" }, diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index 0bb08ce8236..d50e3321275 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -891,23 +891,18 @@ impl JsonRpcHandler { loop { // TODO(optimization): Introduce a view_client method to only get transaction // status without the information about execution outcomes. - match self - .view_client_send(TxStatus { - tx_hash, - signer_account_id: signer_account_id.clone(), - fetch_receipt: false, - }) - .await - { + let signer_account_id = signer_account_id.clone(); + let request = TxStatus { tx_hash, signer_account_id, fetch_receipt: false }; + let tx_status = self.view_client_send(request).await; + match tx_status { // Observed with an execution outcome: the transaction exists. Ok(TxStatusOutcome::Observed(view)) if view.execution_outcome.is_some() => { return Ok(true); } - // Observed without an outcome yet, or shard not tracked (query forwarded): - // keep polling. - Ok( - TxStatusOutcome::Observed(_) | TxStatusOutcome::ShardNotTracked { .. }, - ) => {} + // Observed but no outcome yet: keep polling. + Ok(TxStatusOutcome::Observed(_)) => {} + // Shard not tracked; the query was forwarded: keep polling. + Ok(TxStatusOutcome::DoesNotTrackShard { .. }) => {} // Tracked shard, transaction not found: it does not exist. Ok(TxStatusOutcome::NotObserved) => return Ok(false), Err(err) => last_error = Some(err), @@ -1025,11 +1020,11 @@ impl JsonRpcHandler { // We don't track the shard; the view client forwarded the query. `wait_until: // NONE` says so immediately; otherwise wait for the forwarded answer and, if it // never arrives, time out with the shard recorded. - Ok(TxStatusOutcome::ShardNotTracked { shard_id }) => { + Ok(TxStatusOutcome::DoesNotTrackShard { shard_id }) => { if *finality == TxExecutionStatus::None { ControlFlow::Break(Err(RpcTransactionError::DoesNotTrackShard)) } else { - ControlFlow::Continue(TimeoutErrorCause::ShardNotTracked { shard_id }) + ControlFlow::Continue(TimeoutErrorCause::DoesNotTrackShard { shard_id }) } } // Any other error is terminal; surface it directly rather than waiting. From c8338d5d30c840afcad54b40600ab288a10f7d41 Mon Sep 17 00:00:00 2001 From: wacban Date: Fri, 26 Jun 2026 13:35:44 +0000 Subject: [PATCH 09/14] smarter get_partial_transaction_result --- chain/chain/src/chain.rs | 45 ++++++++++++++++++++----- chain/client/src/view_client_actor.rs | 48 +++++++-------------------- 2 files changed, 48 insertions(+), 45 deletions(-) diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index 8eca86e356c..15681e7aa48 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -3096,12 +3096,37 @@ impl Chain { Ok(FinalExecutionOutcomeView { status, transaction, transaction_outcome, receipts_outcome }) } - /// Returns FinalExecutionOutcomeView for the given transaction. - /// Does not wait for the end of the execution of all corresponding receipts + /// Returns the `FinalExecutionOutcomeView` for the given transaction, + /// assembled from whatever execution outcomes exist so far (it does not + /// wait for all receipts to finish). + /// + /// Errors with `DBNotFoundErr` if the transaction is unknown to this node + /// *or* has no execution outcome yet. Use + /// [`Self::get_partial_transaction_result_option`] when those two cases + /// need to be told apart. pub fn get_partial_transaction_result( &self, transaction_hash: &CryptoHash, ) -> Result { + self.get_partial_transaction_result_option(transaction_hash)?.ok_or_else(|| { + Error::DBNotFoundErr(format!( + "Transaction {} has no execution outcome", + transaction_hash + )) + }) + } + + /// Returns the `Ok(Some(FinalExecutionOutcomeView))` for the given + /// transaction, assembled from whatever execution outcomes exist so far (it + /// does not wait for all receipts to finish). + /// + /// Returns `Ok(None)` if the transaction is recorded on chain but has no + /// execution outcome yet (included, not executed), and `Err(DBNotFoundErr)` + /// if the transaction is not in the store at all. + pub fn get_partial_transaction_result_option( + &self, + transaction_hash: &CryptoHash, + ) -> Result, Error> { let transaction = self.chain_store.get_transaction(transaction_hash).ok_or_else(|| { Error::DBNotFoundErr(format!("Transaction {} is not found", transaction_hash)) })?; @@ -3110,18 +3135,20 @@ impl Chain { let mut outcomes = Vec::new(); self.get_recursive_transaction_results(&mut outcomes, transaction_hash, false)?; if outcomes.is_empty() { - // It can't be, we would fail with tx not found error earlier in this case - // But if so, let's return meaningful error instead of panic on split_off - return Err(Error::DBNotFoundErr(format!( - "Transaction {} is not found", - transaction_hash - ))); + // The transaction is in the store (included in a chunk) but its execution outcome has + // not been recorded yet, so there is no result to assemble. + return Ok(None); } let status = self.get_execution_status(&outcomes, transaction_hash); let receipts_outcome = outcomes.split_off(1); let transaction_outcome = outcomes.pop().unwrap(); - Ok(FinalExecutionOutcomeView { status, transaction, transaction_outcome, receipts_outcome }) + Ok(Some(FinalExecutionOutcomeView { + status, + transaction, + transaction_outcome, + receipts_outcome, + })) } /// Returns corresponding receipts for provided outcome diff --git a/chain/client/src/view_client_actor.rs b/chain/client/src/view_client_actor.rs index 0f7f8f77a5b..fbeb8f6950e 100644 --- a/chain/client/src/view_client_actor.rs +++ b/chain/client/src/view_client_actor.rs @@ -59,10 +59,9 @@ use near_primitives::version::{PROTOCOL_VERSION, ProtocolFeature}; use near_primitives::views::validator_stake_view::ValidatorStakeView; use near_primitives::views::{ BlockView, ChunkView, EpochValidatorInfo, ExecutionOutcomeWithIdView, ExecutionStatusView, - FinalExecutionOutcomeView, FinalExecutionOutcomeViewEnum, FinalExecutionStatus, GasPriceView, - LightClientBlockView, MaintenanceWindowsView, QueryRequest, QueryResponse, ReceiptView, - SignedTransactionView, SplitStorageInfoView, StateChangesKindsView, StateChangesView, - TxExecutionStatus, TxStatusView, + FinalExecutionOutcomeView, FinalExecutionOutcomeViewEnum, GasPriceView, LightClientBlockView, + MaintenanceWindowsView, QueryRequest, QueryResponse, ReceiptView, SplitStorageInfoView, + StateChangesKindsView, StateChangesView, TxExecutionStatus, TxStatusView, }; use near_store::adapter::StoreAdapter as _; use near_store::merkle_proof::MerkleProofAccess; @@ -657,8 +656,8 @@ impl ViewClientActor { .map_err(|err| TxStatusError::InternalError(err.to_string()))?; // Check if we are tracking this shard. if self.shard_tracker.cares_about_shard(&head.prev_block_hash, target_shard_id) { - match self.chain.get_partial_transaction_result(&tx_hash) { - Ok(tx_result) => { + match self.chain.get_partial_transaction_result_option(&tx_hash) { + Ok(Some(tx_result)) => { let status = self.get_tx_execution_status(&tx_result)?; let res = if fetch_receipt { let final_result = @@ -674,36 +673,13 @@ impl ViewClientActor { status, })) } - // TODO: `get_partial_transaction_result` returns `DBNotFoundErr` both when the - // tx is unknown and when its result is merely incomplete; have it return a - // partial result so we don't re-query the store and hardcode a coarse status. - Err(near_chain::Error::DBNotFoundErr(_)) => { - if let Some(transaction) = self.chain.chain_store.get_transaction(&tx_hash) { - let transaction = - SignedTransactionView::from(Arc::unwrap_or_clone(transaction)); - if let Ok(tx_outcome) = self.chain.get_execution_outcome(&tx_hash) { - let outcome = FinalExecutionOutcomeViewEnum::FinalExecutionOutcome( - FinalExecutionOutcomeView { - status: FinalExecutionStatus::Started, - transaction, - transaction_outcome: tx_outcome.into(), - receipts_outcome: vec![], - }, - ); - Ok(TxStatusOutcome::Observed(TxStatusView { - execution_outcome: Some(outcome), - status: TxExecutionStatus::Included, - })) - } else { - Ok(TxStatusOutcome::Observed(TxStatusView { - execution_outcome: None, - status: TxExecutionStatus::Included, - })) - } - } else { - Ok(TxStatusOutcome::NotObserved) - } - } + // The transaction is in the store (included) but has no execution outcome yet. + Ok(None) => Ok(TxStatusOutcome::Observed(TxStatusView { + execution_outcome: None, + status: TxExecutionStatus::Included, + })), + // The transaction is not in this node's store at all. + Err(near_chain::Error::DBNotFoundErr(_)) => Ok(TxStatusOutcome::NotObserved), Err(err) => { tracing::warn!(target: "client", ?err, "error trying to get transaction result"); Err(err.into()) From dec292bfd0eeb476d0d33260c553cf45076424a5 Mon Sep 17 00:00:00 2001 From: wacban Date: Fri, 26 Jun 2026 14:09:03 +0000 Subject: [PATCH 10/14] nits and addressing comments --- chain/client/src/view_client_actor.rs | 19 +++--- .../src/types/transactions.rs | 11 ++++ chain/jsonrpc/src/api/transactions.rs | 4 +- chain/jsonrpc/src/lib.rs | 60 +++++++++---------- 4 files changed, 49 insertions(+), 45 deletions(-) diff --git a/chain/client/src/view_client_actor.rs b/chain/client/src/view_client_actor.rs index fbeb8f6950e..d7c44afced2 100644 --- a/chain/client/src/view_client_actor.rs +++ b/chain/client/src/view_client_actor.rs @@ -668,10 +668,8 @@ impl ViewClientActor { } else { FinalExecutionOutcomeViewEnum::FinalExecutionOutcome(tx_result) }; - Ok(TxStatusOutcome::Observed(TxStatusView { - execution_outcome: Some(res), - status, - })) + let tx_status_view = TxStatusView { execution_outcome: Some(res), status }; + Ok(TxStatusOutcome::Observed(tx_status_view)) } // The transaction is in the store (included) but has no execution outcome yet. Ok(None) => Ok(TxStatusOutcome::Observed(TxStatusView { @@ -1677,14 +1675,13 @@ impl Handler>> for ViewCl metrics::VIEW_CLIENT_MESSAGE_TIME.with_label_values(&["TxStatusRequest"]).start_timer(); let TxStatusRequest { tx_hash, signer_account_id } = msg; let tx_status = self.get_tx_status(tx_hash, signer_account_id, false); - // TODO: the forwarded `TxStatusResponse` only carries the outcome, so the rest of - // `tx_status` (status, `NotObserved`) is dropped. Enrich the response so a forwarded - // query can relay a definitive "not observed" instead of going silent on the RPC node. - let outcome = match tx_status { - Ok(TxStatusOutcome::Observed(view)) => view.execution_outcome, - _ => None, + let Ok(TxStatusOutcome::Observed(view)) = tx_status else { + return None; + }; + let Some(result) = view.execution_outcome else { + return None; }; - outcome.map(|result| Box::new(result.into_outcome())) + Some(Box::new(result.into_outcome())) } } diff --git a/chain/jsonrpc-primitives/src/types/transactions.rs b/chain/jsonrpc-primitives/src/types/transactions.rs index 464b9962dd0..65c8ae5941d 100644 --- a/chain/jsonrpc-primitives/src/types/transactions.rs +++ b/chain/jsonrpc-primitives/src/types/transactions.rs @@ -85,6 +85,17 @@ pub enum TimeoutErrorCause { Error { debug_info: String }, } +impl TimeoutErrorCause { + /// Standardized cause for a plain timeout: the node could not produce a usable transaction + /// status before its polling timeout, with no more specific reason (`NotObserved`, + /// `Pending`, `DoesNotTrackShard`) to report. + pub fn timed_out() -> Self { + Self::Error { + debug_info: "the node timed out before returning a transaction status".to_string(), + } + } +} + #[derive(serde::Serialize, serde::Deserialize, Debug)] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct RpcBroadcastTxSyncResponse { diff --git a/chain/jsonrpc/src/api/transactions.rs b/chain/jsonrpc/src/api/transactions.rs index 8ed44c2d443..8929c82201e 100644 --- a/chain/jsonrpc/src/api/transactions.rs +++ b/chain/jsonrpc/src/api/transactions.rs @@ -68,9 +68,7 @@ impl RpcFrom for RpcTransactionError { Self::UnknownTransaction { requested_transaction_hash } } TxStatusError::InternalError(debug_info) => Self::InternalError { debug_info }, - TxStatusError::TimeoutError => Self::TimeoutError(TimeoutErrorCause::Error { - debug_info: "the node timed out fetching the transaction status".to_string(), - }), + TxStatusError::TimeoutError => Self::TimeoutError(TimeoutErrorCause::timed_out()), } } } diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index d50e3321275..b6a38c8f605 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -946,42 +946,41 @@ impl JsonRpcHandler { > { // If the request times out before any poll completes we report that we never got a // usable status; each poll that keeps us waiting replaces this with a better reason. - let mut timeout_error_cause = TimeoutErrorCause::Error { - debug_info: "the node did not return a transaction status before the request timed out" - .to_string(), - }; + let mut timeout_error_cause = TimeoutErrorCause::timed_out(); - self.clock - .timeout(self.polling_config.polling_timeout, async { - // Create a new watch::Receiver to watch for new blocks. Mark the current block as seen. - let mut new_block_watcher = self.block_notification_watcher.clone(); - new_block_watcher.mark_unchanged(); + let poll_tx_status = async { + // Create a new watch::Receiver to watch for new blocks. Mark the current block as seen. + let mut new_block_watcher = self.block_notification_watcher.clone(); + new_block_watcher.mark_unchanged(); - loop { - match self.poll_tx_status(&tx_info, &finality, fetch_receipt).await { - ControlFlow::Break(outcome) => break outcome, - ControlFlow::Continue(reason) => timeout_error_cause = reason, - } - new_block_watcher.changed().await.map_err(|_| { - RpcTransactionError::InternalError { - debug_info: "block notification channel closed".to_string(), - } - })?; + loop { + match self.check_tx_status(&tx_info, &finality, fetch_receipt).await { + ControlFlow::Break(outcome) => break outcome, + ControlFlow::Continue(reason) => timeout_error_cause = reason, } - }) + new_block_watcher.changed().await.map_err(|_| { + RpcTransactionError::InternalError { + debug_info: "block notification channel closed".to_string(), + } + })?; + } + }; + + // The polling loop returns on its own once it reaches the requested finality or hits a + // definitive error; only a timeout falls through to `unwrap_or_else`. + self.clock + .timeout(self.polling_config.polling_timeout, poll_tx_status) .await - // The polling loop returns on its own once it reaches the requested finality or - // hits a definitive error; only a timeout falls through to `unwrap_or_else`. .unwrap_or_else(|_| { self.tx_status_on_timeout(&tx_info, fetch_receipt, timeout_error_cause) }) } - /// Performs a single `TxStatus` poll for `tx_status_fetch`. Returns `ControlFlow::Break` + /// Runs a single `TxStatus` check for the `poll_tx_status` loop. Returns `ControlFlow::Break` /// with the final response/error once the transaction reaches the requested `finality` /// or hits a definitive error, or `ControlFlow::Continue` with the reason to report if /// the request ultimately times out. - async fn poll_tx_status( + async fn check_tx_status( &self, tx_info: &TransactionInfo, finality: &TxExecutionStatus, @@ -1004,7 +1003,7 @@ impl JsonRpcHandler { // Tracked shard, transaction not on chain. Fail fast if we can prove it invalid; // `wait_until: NONE` reports it unknown immediately, otherwise we keep waiting. Ok(TxStatusOutcome::NotObserved) => { - if let Err(context) = self.validate_tx(tx_info).await { + if let Err(context) = self.detect_invalid_tx(tx_info).await { return ControlFlow::Break(Err(RpcTransactionError::InvalidTransaction { context, })); @@ -1032,12 +1031,11 @@ impl JsonRpcHandler { } } - /// Validates the transaction when we were handed the full signed transaction (rather - /// than just its hash), so the caller can fail fast on an invalid transaction instead - /// of polling for one that will never appear on chain. Returns `Err(context)` when the - /// transaction is known to be invalid, or `Ok(())` when it is valid or we cannot check - /// it (we only have its hash). - async fn validate_tx(&self, tx_info: &TransactionInfo) -> Result<(), InvalidTxError> { + /// Detects an invalid transaction when we were handed the full signed transaction (rather + /// than just its hash), so the caller can fail fast instead of polling for one that will + /// never appear on chain. Returns `Err(context)` when the transaction is known to be + /// invalid, or `Ok(())` when it is valid or we cannot check it (we only have its hash). + async fn detect_invalid_tx(&self, tx_info: &TransactionInfo) -> Result<(), InvalidTxError> { let Some(tx) = tx_info.to_signed_tx() else { return Ok(()) }; match self.send_tx_internal(tx.clone(), true).await { Ok(ProcessTxResponse::InvalidTx(context)) => Err(context), From 496b69efe9cd1d64cca48f20dddb5453fcf9b558 Mon Sep 17 00:00:00 2001 From: wacban Date: Tue, 30 Jun 2026 11:06:31 +0000 Subject: [PATCH 11/14] surface tx submission errors instead of timeouts --- chain/client/src/rpc_handler.rs | 20 ++++++++++--------- chain/jsonrpc/src/lib.rs | 18 +++++++++++------ chain/network/src/client.rs | 7 +++++-- integration-tests/src/env/test_env.rs | 3 ++- .../src/tests/client/process_blocks.rs | 5 +---- .../src/tests/client/query_client.rs | 13 ++++++------ .../src/tests/features/in_memory_tries.rs | 2 +- test-loop-tests/src/utils/transactions.rs | 3 ++- 8 files changed, 41 insertions(+), 30 deletions(-) diff --git a/chain/client/src/rpc_handler.rs b/chain/client/src/rpc_handler.rs index df80fe4984f..4dab0a1818f 100644 --- a/chain/client/src/rpc_handler.rs +++ b/chain/client/src/rpc_handler.rs @@ -26,7 +26,6 @@ use near_primitives::types::AccountId; use near_primitives::types::BlockHeightDelta; use near_primitives::types::EpochId; use near_primitives::types::ShardId; -use near_primitives::unwrap_or_return; use near_primitives::version::ProtocolFeature; use near_store::adapter::StoreAdapter; use near_store::adapter::chain_store::ChainStoreAdapter; @@ -138,12 +137,15 @@ impl RpcHandlerActor { is_forwarded: bool, check_only: bool, ) -> ProcessTxResponse { - unwrap_or_return!(self.process_tx_internal(&tx, is_forwarded, check_only), { - let signer = self.validator_signer.get(); - let me = signer.as_ref().map(|signer| signer.validator_id()); - tracing::debug!(target: "client", ?me, ?tx, "dropping tx"); - ProcessTxResponse::NoResponse - }) + match self.process_tx_internal(&tx, is_forwarded, check_only) { + Ok(response) => response, + Err(err) => { + let signer = self.validator_signer.get(); + let me = signer.as_ref().map(|signer| signer.validator_id()); + tracing::debug!(target: "client", ?me, ?tx, ?err, "failed to process tx"); + ProcessTxResponse::InternalError(err.to_string()) + } + } } /// Process transaction and either add it to the mempool or return to redirect to another validator. @@ -306,7 +308,7 @@ impl RpcHandlerActor { } tracing::trace!(target: "client", %shard_id, tx_hash = ?signed_tx.get_hash(), "non-validator received a forwarded transaction, dropping it"); metrics::TRANSACTION_RECEIVED_NON_VALIDATOR_FORWARDED.inc(); - return Ok(ProcessTxResponse::NoResponse); + return Ok(ProcessTxResponse::Dropped); } if check_only { @@ -315,7 +317,7 @@ impl RpcHandlerActor { if is_forwarded { // Received forwarded transaction but we are not tracking the shard tracing::debug!(target: "client", ?me, %shard_id, tx_hash = ?signed_tx.get_hash(), "received forwarded transaction but no tracking shard"); - return Ok(ProcessTxResponse::NoResponse); + return Ok(ProcessTxResponse::Dropped); } // We are not tracking this shard, so there is no way to validate this tx. Just rerouting. self.forward_tx(&epoch_id, signed_tx).map(|()| ProcessTxResponse::RequestRouted) diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index b6a38c8f605..47cf20072ad 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -351,13 +351,19 @@ impl near_jsonrpc_primitives::types::transactions::RpcTransactionError { pub fn from_network_client_responses(resp: ProcessTxResponse) -> Self { match resp { ProcessTxResponse::InvalidTx(context) => Self::InvalidTransaction { context }, - ProcessTxResponse::NoResponse => Self::TimeoutError(TimeoutErrorCause::Error { - debug_info: "no response from the node".to_string(), - }), - ProcessTxResponse::DoesNotTrackShard | ProcessTxResponse::RequestRouted => { - Self::DoesNotTrackShard + ProcessTxResponse::InternalError(debug_info) => Self::InternalError { debug_info }, + ProcessTxResponse::DoesNotTrackShard => Self::DoesNotTrackShard, + // The node dropped the transaction without processing it. This only happens for + // forwarded transactions, which never reach this direct-send path; surface it as a + // server error rather than a misleading timeout. + ProcessTxResponse::Dropped => Self::InternalError { + debug_info: "the node dropped the transaction without a response".to_string(), + }, + // ValidTx / RequestRouted are successes the caller handles before calling this; + // reaching here is a logic error, so report it instead of silently swallowing it. + resp @ (ProcessTxResponse::ValidTx | ProcessTxResponse::RequestRouted) => { + Self::InternalError { debug_info: format!("unexpected success response: {resp:?}") } } - internal_error => Self::InternalError { debug_info: format!("{:?}", internal_error) }, } } } diff --git a/chain/network/src/client.rs b/chain/network/src/client.rs index 52a9f31d1f7..31c265d99f5 100644 --- a/chain/network/src/client.rs +++ b/chain/network/src/client.rs @@ -121,8 +121,9 @@ pub struct ProcessTxRequest { #[derive(Debug, Clone, PartialEq, Eq)] pub enum ProcessTxResponse { - /// No response. - NoResponse, + /// The node dropped the transaction without acting on it (e.g. a forwarded transaction + /// that this node will neither relay nor include). + Dropped, /// Valid transaction inserted into mempool as response to Transaction. ValidTx, /// Invalid transaction inserted into mempool as response to Transaction. @@ -132,6 +133,8 @@ pub enum ProcessTxResponse { /// The node being queried does not track the shard needed and therefore cannot provide useful /// response. DoesNotTrackShard, + /// Processing the transaction failed with an internal error; the string carries debug context. + InternalError(String), } /// Account announcements that needs to be validated before being processed. diff --git a/integration-tests/src/env/test_env.rs b/integration-tests/src/env/test_env.rs index 125b1f3c993..8a25b584cea 100644 --- a/integration-tests/src/env/test_env.rs +++ b/integration-tests/src/env/test_env.rs @@ -803,11 +803,12 @@ impl TestEnv { let response = self.rpc_handlers[0].process_tx(tx, false, false); // Check if the transaction got rejected match response { - ProcessTxResponse::NoResponse + ProcessTxResponse::Dropped | ProcessTxResponse::RequestRouted | ProcessTxResponse::ValidTx => (), ProcessTxResponse::InvalidTx(e) => return Err(e), ProcessTxResponse::DoesNotTrackShard => panic!("test setup is buggy"), + ProcessTxResponse::InternalError(err) => panic!("process_tx failed: {err}"), } let max_iters = 100; let tip = self.clients[0].chain.head().unwrap(); diff --git a/integration-tests/src/tests/client/process_blocks.rs b/integration-tests/src/tests/client/process_blocks.rs index c7eb3c05381..41b959f1d42 100644 --- a/integration-tests/src/tests/client/process_blocks.rs +++ b/integration-tests/src/tests/client/process_blocks.rs @@ -1300,10 +1300,7 @@ fn test_tx_forwarding_no_double_forwarding() { let tx = env.tx_from_actions(vec![], &signer, signer.get_account_id()); // The transaction has already been forwarded, so it won't be forwarded again. let is_forwarded = true; - assert_eq!( - env.rpc_handlers[0].process_tx(tx, is_forwarded, false), - ProcessTxResponse::NoResponse - ); + assert_eq!(env.rpc_handlers[0].process_tx(tx, is_forwarded, false), ProcessTxResponse::Dropped); assert!(env.network_adapters[0].requests.read().is_empty()); } diff --git a/integration-tests/src/tests/client/query_client.rs b/integration-tests/src/tests/client/query_client.rs index d992a0055f6..3a59057e51e 100644 --- a/integration-tests/src/tests/client/query_client.rs +++ b/integration-tests/src/tests/client/query_client.rs @@ -4,6 +4,7 @@ use near_async::messaging::CanSendAsync; use near_async::time::{Clock, Duration}; use near_client::{ GetBlock, GetBlockWithMerkleTree, GetExecutionOutcomesForBlock, Query, TxStatus, + TxStatusOutcome, }; use near_client_primitives::types::Status; use near_crypto::InMemorySigner; @@ -136,7 +137,7 @@ async fn test_execution_outcome_for_chunk() { assert!(matches!(res, ProcessTxResponse::ValidTx)); tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; - let block_hash = actor_handles + let tx_status_outcome = actor_handles .view_client_actor .send_async(TxStatus { tx_hash, @@ -145,11 +146,11 @@ async fn test_execution_outcome_for_chunk() { }) .await .unwrap() - .unwrap() - .into_outcome() - .unwrap() - .transaction_outcome - .block_hash; + .unwrap(); + let TxStatusOutcome::Observed(tx_status) = tx_status_outcome else { + panic!("expected the transaction to be observed"); + }; + let block_hash = tx_status.into_outcome().unwrap().transaction_outcome.block_hash; let mut execution_outcomes_in_block = actor_handles .view_client_actor diff --git a/integration-tests/src/tests/features/in_memory_tries.rs b/integration-tests/src/tests/features/in_memory_tries.rs index 65da5846d1e..d6da898a738 100644 --- a/integration-tests/src/tests/features/in_memory_tries.rs +++ b/integration-tests/src/tests/features/in_memory_tries.rs @@ -260,7 +260,7 @@ fn run_chain_for_some_blocks_while_sending_money_around( // get a chance to produce the txn if they don't track the shard. for tx_processor in &env.rpc_handlers { match tx_processor.process_tx(txn.clone(), false, false) { - ProcessTxResponse::NoResponse => panic!("No response"), + ProcessTxResponse::Dropped => panic!("transaction was dropped"), ProcessTxResponse::InvalidTx(err) => panic!("Invalid tx: {}", err), _ => {} } diff --git a/test-loop-tests/src/utils/transactions.rs b/test-loop-tests/src/utils/transactions.rs index 568965e12bf..7a00c196425 100644 --- a/test-loop-tests/src/utils/transactions.rs +++ b/test-loop-tests/src/utils/transactions.rs @@ -460,7 +460,7 @@ impl TransactionRunner { } }; let res = match process_tx_response { - ProcessTxResponse::NoResponse => panic!("NoResponse indicates an error"), + ProcessTxResponse::Dropped => panic!("transaction was dropped"), ProcessTxResponse::RequestRouted | // Ok, transaction forwarded to a validator node ProcessTxResponse::ValidTx => TxProcessingResult::Ok, ProcessTxResponse::InvalidTx(err) => match err { @@ -472,6 +472,7 @@ impl TransactionRunner { ProcessTxResponse::DoesNotTrackShard => { panic!("Transaction submitted to a node that doesn't track the shard") } + ProcessTxResponse::InternalError(err) => panic!("process_tx failed: {err}"), }; Some(res) } From a50e5b34d3bbf5db1a9fe66d7dd347675d8ae079 Mon Sep 17 00:00:00 2001 From: wacban Date: Tue, 30 Jun 2026 11:48:11 +0000 Subject: [PATCH 12/14] cleanup --- chain/client-primitives/src/types.rs | 2 -- .../src/types/transactions.rs | 10 ++----- chain/jsonrpc/src/api/transactions.rs | 7 +---- chain/jsonrpc/src/lib.rs | 28 ++++++++++--------- chain/rosetta-rpc/src/errors.rs | 17 +++-------- 5 files changed, 23 insertions(+), 41 deletions(-) diff --git a/chain/client-primitives/src/types.rs b/chain/client-primitives/src/types.rs index e310cb2b13d..75cb81b3c08 100644 --- a/chain/client-primitives/src/types.rs +++ b/chain/client-primitives/src/types.rs @@ -642,9 +642,7 @@ pub enum TxStatusOutcome { #[derive(Debug)] pub enum TxStatusError { ChainError(near_chain_primitives::Error), - MissingTransaction(CryptoHash), InternalError(String), - TimeoutError, } impl From for TxStatusError { diff --git a/chain/jsonrpc-primitives/src/types/transactions.rs b/chain/jsonrpc-primitives/src/types/transactions.rs index 65c8ae5941d..39e43e38e12 100644 --- a/chain/jsonrpc-primitives/src/types/transactions.rs +++ b/chain/jsonrpc-primitives/src/types/transactions.rs @@ -65,8 +65,7 @@ pub struct RpcTransactionResponse { pub final_execution_status: near_primitives::views::TxExecutionStatus, } -/// Explains why a transaction-status request returned a `RpcTransactionError::TimeoutError`: -/// it did not reach the requested `wait_until` finality within the node's polling timeout. +/// Explains why a transaction status request returned a `RpcTransactionError::TimeoutError`: #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] #[serde(tag = "cause", rename_all = "SCREAMING_SNAKE_CASE")] @@ -75,7 +74,6 @@ pub enum TimeoutErrorCause { NotObserved, /// The transaction was observed but is still pending the requested finality. The /// last-known status is included so the caller can re-poll for a higher finality. - /// Boxed to keep `RpcTransactionError` small (it is the `Err` type of many RPC results). Pending { status: Box }, /// The node does not track the transaction's shard and could not get an answer from a /// chunk producer that does before the timeout. @@ -86,10 +84,8 @@ pub enum TimeoutErrorCause { } impl TimeoutErrorCause { - /// Standardized cause for a plain timeout: the node could not produce a usable transaction - /// status before its polling timeout, with no more specific reason (`NotObserved`, - /// `Pending`, `DoesNotTrackShard`) to report. - pub fn timed_out() -> Self { + // A generic timeout error cause when extra context is not available. + pub fn default() -> Self { Self::Error { debug_info: "the node timed out before returning a transaction status".to_string(), } diff --git a/chain/jsonrpc/src/api/transactions.rs b/chain/jsonrpc/src/api/transactions.rs index 8929c82201e..a923f63aa01 100644 --- a/chain/jsonrpc/src/api/transactions.rs +++ b/chain/jsonrpc/src/api/transactions.rs @@ -3,8 +3,7 @@ use near_async::messaging::AsyncSendError; use near_client_primitives::types::TxStatusError; use near_jsonrpc_primitives::errors::RpcParseError; use near_jsonrpc_primitives::types::transactions::{ - RpcSendTransactionRequest, RpcTransactionError, RpcTransactionStatusRequest, TimeoutErrorCause, - TransactionInfo, + RpcSendTransactionRequest, RpcTransactionError, RpcTransactionStatusRequest, TransactionInfo, }; use near_primitives::borsh::BorshDeserialize; use near_primitives::transaction::SignedTransaction; @@ -64,11 +63,7 @@ impl RpcFrom for RpcTransactionError { TxStatusError::ChainError(err) => { Self::InternalError { debug_info: format!("{:?}", err) } } - TxStatusError::MissingTransaction(requested_transaction_hash) => { - Self::UnknownTransaction { requested_transaction_hash } - } TxStatusError::InternalError(debug_info) => Self::InternalError { debug_info }, - TxStatusError::TimeoutError => Self::TimeoutError(TimeoutErrorCause::timed_out()), } } } diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index 47cf20072ad..fdc21825925 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -352,17 +352,16 @@ impl near_jsonrpc_primitives::types::transactions::RpcTransactionError { match resp { ProcessTxResponse::InvalidTx(context) => Self::InvalidTransaction { context }, ProcessTxResponse::InternalError(debug_info) => Self::InternalError { debug_info }, - ProcessTxResponse::DoesNotTrackShard => Self::DoesNotTrackShard, - // The node dropped the transaction without processing it. This only happens for - // forwarded transactions, which never reach this direct-send path; surface it as a - // server error rather than a misleading timeout. ProcessTxResponse::Dropped => Self::InternalError { debug_info: "the node dropped the transaction without a response".to_string(), }, - // ValidTx / RequestRouted are successes the caller handles before calling this; - // reaching here is a logic error, so report it instead of silently swallowing it. - resp @ (ProcessTxResponse::ValidTx | ProcessTxResponse::RequestRouted) => { - Self::InternalError { debug_info: format!("unexpected success response: {resp:?}") } + ProcessTxResponse::DoesNotTrackShard => Self::DoesNotTrackShard, + // ValidTx / RequestRouted are successes so this path should not be + // reachable. Return internal error to avoid panicking. + response @ (ProcessTxResponse::ValidTx | ProcessTxResponse::RequestRouted) => { + Self::InternalError { + debug_info: format!("unexpected success response: {response:?}"), + } } } } @@ -952,7 +951,7 @@ impl JsonRpcHandler { > { // If the request times out before any poll completes we report that we never got a // usable status; each poll that keeps us waiting replaces this with a better reason. - let mut timeout_error_cause = TimeoutErrorCause::timed_out(); + let mut timeout_error_cause = TimeoutErrorCause::default(); let poll_tx_status = async { // Create a new watch::Receiver to watch for new blocks. Mark the current block as seen. @@ -982,10 +981,13 @@ impl JsonRpcHandler { }) } - /// Runs a single `TxStatus` check for the `poll_tx_status` loop. Returns `ControlFlow::Break` - /// with the final response/error once the transaction reaches the requested `finality` - /// or hits a definitive error, or `ControlFlow::Continue` with the reason to report if - /// the request ultimately times out. + /// Runs a single `TxStatus` check for the `poll_tx_status` loop. + /// + /// Returns `ControlFlow::Break` with the final response/error once the + /// transaction reaches the requested `finality` or hits a definitive error. + /// + /// Returns `ControlFlow::Continue` with the reason to report if the request + /// ultimately times out. async fn check_tx_status( &self, tx_info: &TransactionInfo, diff --git a/chain/rosetta-rpc/src/errors.rs b/chain/rosetta-rpc/src/errors.rs index 38567297003..24b208833a6 100644 --- a/chain/rosetta-rpc/src/errors.rs +++ b/chain/rosetta-rpc/src/errors.rs @@ -42,19 +42,10 @@ impl From for ErrorKind { "Transaction could not be found due to an internal error: {:?}", err )), - near_client::TxStatusError::MissingTransaction(err) => { - Self::NotFound(format!("Transaction is missing: {:?}", err)) - } - near_client::TxStatusError::InternalError(_) - | near_client::TxStatusError::TimeoutError => { - // TODO: remove the statuses from TxStatusError since they are - // never constructed by the view client (it is a leak of - // abstraction introduced in JSONRPC) - Self::InternalInvariantError(format!( - "TxStatusError reached unexpected state: {:?}", - err - )) - } + near_client::TxStatusError::InternalError(_) => Self::InternalInvariantError(format!( + "TxStatusError reached unexpected state: {:?}", + err + )), } } } From db7c65e71fb3fe29b54246606fc0ab30b75d8b87 Mon Sep 17 00:00:00 2001 From: wacban Date: Tue, 30 Jun 2026 12:36:43 +0000 Subject: [PATCH 13/14] minor --- chain/jsonrpc/src/lib.rs | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index fdc21825925..747bd716b96 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -959,7 +959,7 @@ impl JsonRpcHandler { new_block_watcher.mark_unchanged(); loop { - match self.check_tx_status(&tx_info, &finality, fetch_receipt).await { + match self.tx_status_fetch_single(&tx_info, &finality, fetch_receipt).await { ControlFlow::Break(outcome) => break outcome, ControlFlow::Continue(reason) => timeout_error_cause = reason, } @@ -988,7 +988,7 @@ impl JsonRpcHandler { /// /// Returns `ControlFlow::Continue` with the reason to report if the request /// ultimately times out. - async fn check_tx_status( + async fn tx_status_fetch_single( &self, tx_info: &TransactionInfo, finality: &TxExecutionStatus, @@ -1039,22 +1039,6 @@ impl JsonRpcHandler { } } - /// Detects an invalid transaction when we were handed the full signed transaction (rather - /// than just its hash), so the caller can fail fast instead of polling for one that will - /// never appear on chain. Returns `Err(context)` when the transaction is known to be - /// invalid, or `Ok(())` when it is valid or we cannot check it (we only have its hash). - async fn detect_invalid_tx(&self, tx_info: &TransactionInfo) -> Result<(), InvalidTxError> { - let Some(tx) = tx_info.to_signed_tx() else { return Ok(()) }; - match self.send_tx_internal(tx.clone(), true).await { - Ok(ProcessTxResponse::InvalidTx(context)) => Err(context), - _ => Ok(()), - } - } - - /// Builds the `TimeoutError` returned when `tx_status_fetch`'s polling loop is cancelled - /// by the timeout. The `cause` records why the request did not reach the requested - /// finality in time (see `TimeoutErrorCause`), so the caller still has full - /// information and can re-poll. fn tx_status_on_timeout( &self, tx_info: &TransactionInfo, @@ -1073,6 +1057,18 @@ impl JsonRpcHandler { Err(RpcTransactionError::TimeoutError(cause)) } + /// Detects an invalid transaction when we were handed the full signed transaction (rather + /// than just its hash), so the caller can fail fast instead of polling for one that will + /// never appear on chain. Returns `Err(context)` when the transaction is known to be + /// invalid, or `Ok(())` when it is valid or we cannot check it (we only have its hash). + async fn detect_invalid_tx(&self, tx_info: &TransactionInfo) -> Result<(), InvalidTxError> { + let Some(tx) = tx_info.to_signed_tx() else { return Ok(()) }; + match self.send_tx_internal(tx.clone(), true).await { + Ok(ProcessTxResponse::InvalidTx(context)) => Err(context), + _ => Ok(()), + } + } + /// Send a transaction idempotently (subsequent send of the same transaction will not cause /// any new side-effects and the result will be the same unless we garbage collected it /// already). From b4bf221a44db73bb5fba06e8307d5b1ce625c0d8 Mon Sep 17 00:00:00 2001 From: wacban Date: Tue, 30 Jun 2026 13:18:46 +0000 Subject: [PATCH 14/14] clippy & openAPI --- chain/client-primitives/src/types.rs | 3 ++- chain/client/src/view_client_actor.rs | 11 +++++++---- chain/jsonrpc/openapi/openapi.json | 4 ++-- chain/jsonrpc/src/lib.rs | 4 ++-- integration-tests/src/tests/client/query_client.rs | 2 +- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/chain/client-primitives/src/types.rs b/chain/client-primitives/src/types.rs index 75cb81b3c08..6413019efa3 100644 --- a/chain/client-primitives/src/types.rs +++ b/chain/client-primitives/src/types.rs @@ -631,7 +631,8 @@ pub struct TxStatus { #[derive(Debug)] pub enum TxStatusOutcome { /// The node tracks the transaction's shard and observed it. - Observed(TxStatusView), + /// Boxed to keep the enum small (`TxStatusView` is large; the other variants are tiny). + Observed(Box), /// The node tracks the shard but has not seen the transaction on chain. NotObserved, /// The node does not track the transaction's shard; the query was forwarded diff --git a/chain/client/src/view_client_actor.rs b/chain/client/src/view_client_actor.rs index d7c44afced2..39581760191 100644 --- a/chain/client/src/view_client_actor.rs +++ b/chain/client/src/view_client_actor.rs @@ -646,7 +646,10 @@ impl ViewClientActor { let status = self.get_tx_execution_status(&res)?; let execution_outcome = Some(FinalExecutionOutcomeViewEnum::FinalExecutionOutcome(res)); - return Ok(TxStatusOutcome::Observed(TxStatusView { execution_outcome, status })); + return Ok(TxStatusOutcome::Observed(Box::new(TxStatusView { + execution_outcome, + status, + }))); } } @@ -669,13 +672,13 @@ impl ViewClientActor { FinalExecutionOutcomeViewEnum::FinalExecutionOutcome(tx_result) }; let tx_status_view = TxStatusView { execution_outcome: Some(res), status }; - Ok(TxStatusOutcome::Observed(tx_status_view)) + Ok(TxStatusOutcome::Observed(Box::new(tx_status_view))) } // The transaction is in the store (included) but has no execution outcome yet. - Ok(None) => Ok(TxStatusOutcome::Observed(TxStatusView { + Ok(None) => Ok(TxStatusOutcome::Observed(Box::new(TxStatusView { execution_outcome: None, status: TxExecutionStatus::Included, - })), + }))), // The transaction is not in this node's store at all. Err(near_chain::Error::DBNotFoundErr(_)) => Ok(TxStatusOutcome::NotObserved), Err(err) => { diff --git a/chain/jsonrpc/openapi/openapi.json b/chain/jsonrpc/openapi/openapi.json index 04ea5e3a820..5c8aa513a1e 100644 --- a/chain/jsonrpc/openapi/openapi.json +++ b/chain/jsonrpc/openapi/openapi.json @@ -20804,7 +20804,7 @@ "type": "object" }, "TimeoutErrorCause": { - "description": "Explains why a transaction-status request returned a `RpcTransactionError::TimeoutError`:\nit did not reach the requested `wait_until` finality within the node's polling timeout.", + "description": "Explains why a transaction status request returned a `RpcTransactionError::TimeoutError`:", "oneOf": [ { "description": "The node never observed the transaction on chain.", @@ -20822,7 +20822,7 @@ "type": "object" }, { - "description": "The transaction was observed but is still pending the requested finality. The\nlast-known status is included so the caller can re-poll for a higher finality.\nBoxed to keep `RpcTransactionError` small (it is the `Err` type of many RPC results).", + "description": "The transaction was observed but is still pending the requested finality. The\nlast-known status is included so the caller can re-poll for a higher finality.", "properties": { "cause": { "enum": [ diff --git a/chain/jsonrpc/src/lib.rs b/chain/jsonrpc/src/lib.rs index 747bd716b96..3e7c33d0d38 100644 --- a/chain/jsonrpc/src/lib.rs +++ b/chain/jsonrpc/src/lib.rs @@ -1001,10 +1001,10 @@ impl JsonRpcHandler { // the requested finality; otherwise keep polling, remembering how far it got. Ok(TxStatusOutcome::Observed(view)) => { if tx_execution_status_meets_expectations(finality, &view.status) { - ControlFlow::Break(Ok(view.into())) + ControlFlow::Break(Ok((*view).into())) } else { ControlFlow::Continue(TimeoutErrorCause::Pending { - status: Box::new(view.into()), + status: Box::new((*view).into()), }) } } diff --git a/integration-tests/src/tests/client/query_client.rs b/integration-tests/src/tests/client/query_client.rs index 3a59057e51e..9730c7a68a7 100644 --- a/integration-tests/src/tests/client/query_client.rs +++ b/integration-tests/src/tests/client/query_client.rs @@ -150,7 +150,7 @@ async fn test_execution_outcome_for_chunk() { let TxStatusOutcome::Observed(tx_status) = tx_status_outcome else { panic!("expected the transaction to be observed"); }; - let block_hash = tx_status.into_outcome().unwrap().transaction_outcome.block_hash; + let block_hash = (*tx_status).into_outcome().unwrap().transaction_outcome.block_hash; let mut execution_outcomes_in_block = actor_handles .view_client_actor