diff --git a/Cargo.toml b/Cargo.toml index f3a6b1d..25185a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,10 @@ [workspace] -members = ["solana/account", "solana/transaction-context", "solana/transaction-view"] +members = [ + "solana/account", + "solana/program-runtime", + "solana/transaction-context", + "solana/transaction-view", +] resolver = "3" [workspace.package] @@ -15,16 +20,22 @@ version = "0.1.0" magic-root-interface = { path = "programs/magic-root-interface" } magic-root-program = { path = "programs/magic-root-program" } solana-account = { path = "solana/account" } +solana-program-runtime = { path = "solana/program-runtime" } solana-transaction-context = { path = "solana/transaction-context" } ahash = "0.8.12" arc-swap = "1.9.1" +assert_matches = "1.5.0" +base64 = "0.22.1" bincode = "1.3.3" bitflags = "2.11.1" blake3 = "1.8.5" +cfg-if = "1.0.4" criterion = "0.8.2" +derive_more = "2.1.1" +itertools = "0.13.0" qualifier_attr = "0.2.2" -rand = "0.8.5" +rand = "0.9.2" rustix = { version = "1.1.4" } serde = "1.0.228" serde_bytes = "0.11.19" @@ -45,20 +56,44 @@ solana-account-info = "3.1.1" solana-clock = "3.1.0" solana-compute-budget-instruction = "=4.1.1" solana-cpi = "3.1.0" +solana-epoch-rewards = "3.0.1" +solana-epoch-schedule = "3.1.0" +solana-feature-gate-interface = { version = "4.0.0", features = ["bincode"] } +solana-fee-structure = "3.0.0" solana-hash = "4.3.0" solana-instruction = "3.4.0" solana-instruction-error = "2.3.0" solana-instructions-sysvar = "4.0.0" +solana-keypair = "3.1.2" +solana-last-restart-slot = "3.0.0" +solana-loader-v3-interface = "6.1.0" solana-program-entrypoint = "3.1.1" solana-program-error = "3.0.1" solana-pubkey = "4.2.0" -solana-rent = "4.0.0-rc.1" -solana-sbpf = "0.13.1" +solana-rent = "4.2.0" +solana-sbpf = "=0.21.0" solana-sdk-ids = "3.1.0" solana-short-vec = "=3.2.2" solana-signature = "=3.4.1" -solana-sysvar = "4.0.0" -solana-system-interface = "3.2.0" +solana-signer = "3.0.1" +solana-slot-hashes = "3.0.1" +solana-stable-layout = "3.0.0" +solana-sysvar = "3.1.1" +solana-sysvar-id = "3.1.0" +solana-system-interface = ">=3.0.0, <3.2.0" +solana-svm-callback = "4.0.0-rc.1" +solana-svm-feature-set = "4.0.0-rc.1" +solana-svm-log-collector = "4.0.0-rc.1" +solana-svm-measure = "4.0.0-rc.1" +solana-svm-timings = "4.0.0-rc.1" +solana-svm-transaction = "4.0.0-rc.1" +solana-svm-type-overrides = "4.0.0-rc.1" +solana-system-interface = ">=3.0.0, <3.2.0" +solana-system-program = "4.0.0-rc.0" +solana-system-transaction = "3.0.0" +solana-sysvar = "3.1.1" +solana-sysvar-id = "3.1.0" +solana-transaction = "4.1.1" solana-transaction-error = "3.2.0" [patch.crates-io] diff --git a/solana/program-runtime/Cargo.toml b/solana/program-runtime/Cargo.toml index a2670bc..a213e3f 100644 --- a/solana/program-runtime/Cargo.toml +++ b/solana/program-runtime/Cargo.toml @@ -1,5 +1,7 @@ [package] name = "solana-program-runtime" + +authors = { workspace = true } description = "Solana program runtime" documentation = "https://docs.rs/solana-program-runtime" version = { workspace = true } @@ -7,7 +9,8 @@ authors = { workspace = true } repository = { workspace = true } homepage = { workspace = true } license = { workspace = true } -edition = "2024" +repository = { workspace = true } +version = "4.1.1" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] @@ -17,9 +20,12 @@ crate-type = ["lib"] name = "solana_program_runtime" [features] +# No-op stub retained only so external (patched-in) crates that reference +# `solana-program-runtime/agave-unstable-api` still resolve; the lib is no +# longer gated on it, and the upstream svm-* deps enable their unstable API +# unconditionally below. agave-unstable-api = [] -conf-stack-frame-size = ["solana-sbpf/conf-stack-frame-size"] -dev-context-only-utils = ["dep:solana-message"] +dev-context-only-utils = [] dummy-for-ci-check = ["metrics"] # Compatibility stub: this fork does not derive or consume frozen ABI metadata. frozen-abi = [] @@ -33,8 +39,8 @@ base64 = { workspace = true } bincode = { workspace = true } cfg-if = { workspace = true } itertools = { workspace = true } -percentage = { workspace = true } qualifier_attr = { workspace = true, optional = true } +scc = { workspace = true } serde = { workspace = true } solana-account = { workspace = true, features = ["bincode"] } solana-account-info = { workspace = true } @@ -42,17 +48,10 @@ solana-clock = { workspace = true } solana-epoch-rewards = { workspace = true } solana-epoch-schedule = { workspace = true } solana-fee-structure = { workspace = true } -solana-frozen-abi = { workspace = true, optional = true, features = [ - "frozen-abi", -] } -solana-frozen-abi-macro = { workspace = true, optional = true, features = [ - "frozen-abi", -] } solana-hash = { workspace = true } solana-instruction = { workspace = true } solana-last-restart-slot = { workspace = true } solana-loader-v3-interface = { workspace = true } -solana-message = { workspace = true, optional = true } solana-program-entrypoint = { workspace = true } solana-pubkey = { workspace = true } solana-rent = { workspace = true } @@ -60,14 +59,13 @@ solana-sbpf = { workspace = true, features = ["jit"] } solana-sdk-ids = { workspace = true } solana-slot-hashes = { workspace = true } solana-stable-layout = { workspace = true } -solana-stake-interface = { workspace = true, features = ["bincode", "sysvar"] } -solana-svm-callback = { workspace = true } -solana-svm-feature-set = { workspace = true } -solana-svm-log-collector = { workspace = true } -solana-svm-measure = { workspace = true } -solana-svm-timings = { workspace = true } -solana-svm-transaction = { workspace = true } -solana-svm-type-overrides = { workspace = true } +solana-svm-callback = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-feature-set = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-log-collector = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-measure = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-timings = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-transaction = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-type-overrides = { workspace = true, features = ["agave-unstable-api"] } solana-system-interface = { workspace = true } solana-sysvar = { workspace = true, features = ["bincode"] } solana-sysvar-id = { workspace = true } @@ -83,12 +81,11 @@ solana-program-runtime = { path = ".", features = ["dev-context-only-utils"] } solana-pubkey = { workspace = true, features = ["rand"] } solana-signer = { workspace = true } solana-transaction = { workspace = true, features = ["dev-context-only-utils"] } -solana-transaction-context = { path = "../transaction-context", features = [ - "agave-unstable-api", - "bincode", - "dev-context-only-utils", -] } -test-case = { workspace = true } +solana-transaction-context = { workspace = true, features = ["dev-context-only-utils"] } +test-case = "3.3.1" + +[lints.rust] +unexpected_cfgs = "allow" -[lints] -workspace = true +[lints.clippy] +too_many_arguments = "allow" diff --git a/solana/program-runtime/README.md b/solana/program-runtime/README.md new file mode 100644 index 0000000..2c351ac --- /dev/null +++ b/solana/program-runtime/README.md @@ -0,0 +1,16 @@ +# `solana-program-runtime` + +This Agave fork implements invocation state, CPI translation, SBF VM setup, +sysvar access, logging, serialization, and program-cache primitives. Workspace +`[patch.crates-io]` entries force the dependency graph to use this copy. + +Account loading and transaction-level policy belong to `solana-svm`. The +engine-specific direct account mapping, access-violation growth, and CPI +synchronization contracts are documented in [`../README.md`](../README.md). + +Changes to `serialization`, CPI account-region replacement, or `vm` error +mapping must remain synchronized with the transaction-context access-violation +handler. + +The `frozen-abi` feature is retained as a no-op compatibility stub; this fork +does not derive or consume frozen ABI metadata. diff --git a/solana/program-runtime/fixtures/noop_aligned.so b/solana/program-runtime/fixtures/noop_aligned.so new file mode 100644 index 0000000..f00ca36 Binary files /dev/null and b/solana/program-runtime/fixtures/noop_aligned.so differ diff --git a/solana/program-runtime/src/cpi.rs b/solana/program-runtime/src/cpi.rs index 98a9631..362c615 100644 --- a/solana/program-runtime/src/cpi.rs +++ b/solana/program-runtime/src/cpi.rs @@ -1,11 +1,10 @@ -//! Cross-Program Invocation (CPI) error types +//! Cross-program invocation translation and dispatch. use { crate::{ - invoke_context::InvokeContext, + invoke_context::{InvokeContext, SerializedAccountMetadata}, memory::{translate_slice, translate_type, translate_type_mut_for_cpi, translate_vm_slice}, - memory_context::SerializedAccountMetadata, - serialization::{create_memory_region_of_account, modify_memory_region_of_account}, + serialization::create_memory_region_of_account, }, solana_account_info::AccountInfo, solana_instruction::{AccountMeta, Instruction, error::InstructionError}, @@ -16,7 +15,7 @@ use { solana_sdk_ids::{bpf_loader, bpf_loader_deprecated, native_loader}, solana_stable_layout::stable_instruction::StableInstruction, solana_svm_log_collector::ic_msg, - solana_svm_timings::ExecuteTimings, + solana_svm_measure::measure::Measure, solana_transaction_context::{ IndexOfAccount, MAX_ACCOUNTS_PER_INSTRUCTION, MAX_INSTRUCTION_DATA_LEN, instruction_accounts::BorrowedInstructionAccount, vm_slice::VmSlice, @@ -25,7 +24,7 @@ use { thiserror::Error, }; -/// CPI-specific error types +/// Errors produced while translating or dispatching a CPI request. #[derive(Debug, Error, PartialEq, Eq)] pub enum CpiError { #[error("Invalid pointer")] @@ -37,20 +36,14 @@ pub enum CpiError { #[error("InvalidLength")] InvalidLength, #[error("Invoked an instruction with too many accounts ({num_accounts} > {max_accounts})")] - MaxInstructionAccountsExceeded { - num_accounts: u64, - max_accounts: u64, - }, + MaxInstructionAccountsExceeded { num_accounts: u64, max_accounts: u64 }, #[error("Invoked an instruction with data that is too large ({data_len} > {max_data_len})")] MaxInstructionDataLenExceeded { data_len: u64, max_data_len: u64 }, #[error( "Invoked an instruction with too many account info's ({num_account_infos} > \ {max_account_infos})" )] - MaxInstructionAccountInfosExceeded { - num_account_infos: u64, - max_account_infos: u64, - }, + MaxInstructionAccountInfosExceeded { num_account_infos: u64, max_account_infos: u64 }, #[error("Program {0} not supported by inner instructions")] ProgramNotSupported(Pubkey), } @@ -58,9 +51,11 @@ pub enum CpiError { type Error = Box; const SUCCESS: u64 = 0; -/// Maximum signers +/// Maximum signer seed groups accepted by CPI. const MAX_SIGNERS: usize = 16; -///SIMD-0339 based calculation of AccountInfo translation byte size. Fixed size of **80 bytes** for each AccountInfo broken down as: +/// SIMD-0339 `AccountInfo` translation byte size. +/// +/// Fixed size of **80 bytes** for each AccountInfo, broken down as: /// - 32 bytes for account address /// - 32 bytes for owner address /// - 8 bytes for lamport balance @@ -119,7 +114,9 @@ struct SolSignerSeedsC { } /// Maximum number of account info structs that can be used in a single CPI invocation -const MAX_CPI_ACCOUNT_INFOS: usize = 255; +const MAX_CPI_ACCOUNT_INFOS: usize = 128; +/// Maximum number of account info structs that can be used in a single CPI invocation with SIMD-0339 active +const MAX_CPI_ACCOUNT_INFOS_SIMD_0339: usize = 255; /// Check that an account info pointer field points to the expected address fn check_account_info_pointer( @@ -159,9 +156,20 @@ fn check_instruction_size(num_accounts: usize, data_len: usize) -> Result<(), Er } /// Check that the number of account infos is within the CPI limit -fn check_account_infos(num_account_infos: usize) -> Result<(), Error> { +fn check_account_infos( + num_account_infos: usize, + invoke_context: &InvokeContext, +) -> Result<(), Error> { + let max_cpi_account_infos = if invoke_context.get_feature_set().increase_cpi_account_info_limit + { + MAX_CPI_ACCOUNT_INFOS_SIMD_0339 + } else if invoke_context.get_feature_set().increase_tx_account_lock_limit { + MAX_CPI_ACCOUNT_INFOS + } else { + 64 + }; let num_account_infos = num_account_infos as u64; - let max_account_infos = MAX_CPI_ACCOUNT_INFOS as u64; + let max_account_infos = max_cpi_account_infos as u64; if num_account_infos > max_account_infos { return Err(Box::new(CpiError::MaxInstructionAccountInfosExceeded { num_account_infos, @@ -183,9 +191,7 @@ fn check_authorized_program( || (solana_sdk_ids::bpf_loader_upgradeable::check_id(program_id) && !(bpf_loader_upgradeable::is_upgrade_instruction(instruction_data) || bpf_loader_upgradeable::is_set_authority_instruction(instruction_data) - || (invoke_context - .get_feature_set() - .enable_bpf_loader_set_authority_checked_ix + || (invoke_context.get_feature_set().enable_bpf_loader_set_authority_checked_ix && bpf_loader_upgradeable::is_set_authority_checked_instruction( instruction_data, )) @@ -214,7 +220,7 @@ pub struct CallerAccount<'a> { // mapped inside the vm (see serialize_parameters() in // BpfExecutor::execute). // - // This is only set when account_data_direct_mapping is off. + // Empty when account data is directly mapped. pub serialized_data: &'a mut [u8], // Given the corresponding input AccountInfo::data, vm_data_addr points to // the pointer field and ref_to_len_in_vm points to the length field. @@ -224,90 +230,44 @@ pub struct CallerAccount<'a> { impl<'a> CallerAccount<'a> { pub fn get_serialized_data( - memory_mapping: &solana_sbpf::memory_region::MemoryMapping, check_aligned: bool, - vm_addr: u64, original_data_len: usize, len: usize, - syscall_parameter_address_restrictions: bool, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, ) -> Result<&'a mut [u8], Error> { - use crate::memory::translate_slice_mut_for_cpi; - - if syscall_parameter_address_restrictions { - let is_caller_loader_deprecated = !check_aligned; - let address_space_reserved_for_account = if is_caller_loader_deprecated { - original_data_len - } else { - original_data_len.saturating_add(MAX_PERMITTED_DATA_INCREASE) - }; - if len > address_space_reserved_for_account { - return Err(InstructionError::InvalidRealloc.into()); - } - } - if virtual_address_space_adjustments && account_data_direct_mapping { - Ok(&mut []) - } else if virtual_address_space_adjustments { - // Workaround the memory permissions (as these are from the PoV of being inside the VM) - let serialization_ptr = translate_slice_mut_for_cpi::( - memory_mapping, - solana_sbpf::ebpf::MM_INPUT_START, - 1, - false, // Don't care since it is byte aligned - )? - .as_mut_ptr(); - unsafe { - Ok(std::slice::from_raw_parts_mut( - serialization_ptr - .add(vm_addr.saturating_sub(solana_sbpf::ebpf::MM_INPUT_START) as usize), - len, - )) - } + let is_caller_loader_deprecated = !check_aligned; + let address_space_reserved_for_account = if is_caller_loader_deprecated { + original_data_len } else { - translate_slice_mut_for_cpi::( - memory_mapping, - vm_addr, - len as u64, - false, // Don't care since it is byte aligned - ) + original_data_len.saturating_add(MAX_PERMITTED_DATA_INCREASE) + }; + if len > address_space_reserved_for_account { + return Err(InstructionError::InvalidRealloc.into()); } + Ok(&mut []) } // Create a CallerAccount given an AccountInfo. pub fn from_account_info( invoke_context: &InvokeContext, - memory_mapping: &MemoryMapping, + memory_mapping: &solana_sbpf::memory_region::MemoryMapping, check_aligned: bool, - _vm_addr: u64, account_info: &solana_account_info::AccountInfo, - account_metadata: &crate::memory_context::SerializedAccountMetadata, + account_metadata: &crate::invoke_context::SerializedAccountMetadata, ) -> Result, Error> { use crate::memory::{translate_type, translate_type_mut_for_cpi}; - let syscall_parameter_address_restrictions = invoke_context - .get_feature_set() - .syscall_parameter_address_restrictions; - let virtual_address_space_adjustments = invoke_context - .get_feature_set() - .virtual_address_space_adjustments; - let account_data_direct_mapping = - invoke_context.get_feature_set().account_data_direct_mapping; - - if syscall_parameter_address_restrictions { - check_account_info_pointer( - invoke_context, - account_info.key as *const _ as u64, - account_metadata.vm_key_addr, - "key", - )?; - check_account_info_pointer( - invoke_context, - account_info.owner as *const _ as u64, - account_metadata.vm_owner_addr, - "owner", - )?; - } + check_account_info_pointer( + invoke_context, + account_info.key as *const _ as u64, + account_metadata.vm_key_addr, + "key", + )?; + check_account_info_pointer( + invoke_context, + account_info.owner as *const _ as u64, + account_metadata.vm_owner_addr, + "owner", + )?; // account_info points to host memory. The addresses used internally are // in vm space so they need to be translated. @@ -318,18 +278,16 @@ impl<'a> CallerAccount<'a> { account_info.lamports.as_ptr() as u64, check_aligned, )?; - if syscall_parameter_address_restrictions { - if account_info.lamports.as_ptr() as u64 >= solana_sbpf::ebpf::MM_INPUT_START { - return Err(Box::new(CpiError::InvalidPointer)); - } - - check_account_info_pointer( - invoke_context, - *ptr, - account_metadata.vm_lamports_addr, - "lamports", - )?; + if account_info.lamports.as_ptr() as u64 >= solana_sbpf::ebpf::MM_INPUT_START { + return Err(Box::new(CpiError::InvalidPointer)); } + + check_account_info_pointer( + invoke_context, + *ptr, + account_metadata.vm_lamports_addr, + "lamports", + )?; translate_type_mut_for_cpi::(memory_mapping, *ptr, check_aligned)? }; @@ -340,9 +298,7 @@ impl<'a> CallerAccount<'a> { )?; let (serialized_data, vm_data_addr, ref_to_len_in_vm) = { - if syscall_parameter_address_restrictions - && account_info.data.as_ptr() as u64 >= solana_sbpf::ebpf::MM_INPUT_START - { + if account_info.data.as_ptr() as u64 >= solana_sbpf::ebpf::MM_INPUT_START { return Err(Box::new(CpiError::InvalidPointer)); } @@ -352,48 +308,28 @@ impl<'a> CallerAccount<'a> { account_info.data.as_ptr() as *const _ as u64, check_aligned, )?; - if syscall_parameter_address_restrictions { - check_account_info_pointer( - invoke_context, - data.as_ptr() as u64, - account_metadata.vm_data_addr, - "data", - )?; - } else { - // Moved to translate_accounts_common() via feature gate. - invoke_context.compute_meter.consume_checked( - (data.len() as u64) - .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) - .unwrap_or(u64::MAX), - )?; - } + check_account_info_pointer( + invoke_context, + data.as_ptr() as u64, + account_metadata.vm_data_addr, + "data", + )?; let vm_len_addr = (account_info.data.as_ptr() as *const u64 as u64) .saturating_add(size_of::() as u64); - if syscall_parameter_address_restrictions { - // In the same vein as the other check_account_info_pointer() checks, we don't lock - // this pointer to a specific address but we don't want it to be inside accounts, or - // callees might be able to write to the pointed memory. - if vm_len_addr >= solana_sbpf::ebpf::MM_INPUT_START { - return Err(Box::new(CpiError::InvalidPointer)); - } + // In the same vein as the other check_account_info_pointer() checks, we don't lock + // this pointer to a specific address but we don't want it to be inside accounts, or + // callees might be able to write to the pointed memory. + if vm_len_addr >= solana_sbpf::ebpf::MM_INPUT_START { + return Err(Box::new(CpiError::InvalidPointer)); } let ref_to_len_in_vm = translate_type_mut_for_cpi::(memory_mapping, vm_len_addr, false)?; let vm_data_addr = data.as_ptr() as u64; let serialized_data = CallerAccount::get_serialized_data( - memory_mapping, check_aligned, - vm_data_addr, account_metadata.original_data_len, - if syscall_parameter_address_restrictions { - *ref_to_len_in_vm as usize - } else { - data.len() - }, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, + *ref_to_len_in_vm as usize, )?; (serialized_data, vm_data_addr, ref_to_len_in_vm) }; @@ -411,52 +347,41 @@ impl<'a> CallerAccount<'a> { // Create a CallerAccount given a SolAccountInfo. fn from_sol_account_info( invoke_context: &InvokeContext, - memory_mapping: &MemoryMapping, + memory_mapping: &solana_sbpf::memory_region::MemoryMapping, check_aligned: bool, vm_addr: u64, account_info: &SolAccountInfo, - account_metadata: &crate::memory_context::SerializedAccountMetadata, + account_metadata: &crate::invoke_context::SerializedAccountMetadata, ) -> Result, Error> { use crate::memory::translate_type_mut_for_cpi; - let syscall_parameter_address_restrictions = invoke_context - .get_feature_set() - .syscall_parameter_address_restrictions; - let virtual_address_space_adjustments = invoke_context - .get_feature_set() - .virtual_address_space_adjustments; - let account_data_direct_mapping = - invoke_context.get_feature_set().account_data_direct_mapping; - - if syscall_parameter_address_restrictions { - check_account_info_pointer( - invoke_context, - account_info.key_addr, - account_metadata.vm_key_addr, - "key", - )?; + check_account_info_pointer( + invoke_context, + account_info.key_addr, + account_metadata.vm_key_addr, + "key", + )?; - check_account_info_pointer( - invoke_context, - account_info.owner_addr, - account_metadata.vm_owner_addr, - "owner", - )?; + check_account_info_pointer( + invoke_context, + account_info.owner_addr, + account_metadata.vm_owner_addr, + "owner", + )?; - check_account_info_pointer( - invoke_context, - account_info.lamports_addr, - account_metadata.vm_lamports_addr, - "lamports", - )?; + check_account_info_pointer( + invoke_context, + account_info.lamports_addr, + account_metadata.vm_lamports_addr, + "lamports", + )?; - check_account_info_pointer( - invoke_context, - account_info.data_addr, - account_metadata.vm_data_addr, - "data", - )?; - } + check_account_info_pointer( + invoke_context, + account_info.data_addr, + account_metadata.vm_data_addr, + "data", + )?; // account_info points to host memory. The addresses used internally are // in vm space so they need to be translated. @@ -471,16 +396,6 @@ impl<'a> CallerAccount<'a> { check_aligned, )?; - if !syscall_parameter_address_restrictions { - // Moved to translate_accounts_common() via feature gate. - invoke_context.compute_meter.consume_checked( - account_info - .data_len - .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) - .unwrap_or(u64::MAX), - )?; - } - // we already have the host addr we want: &mut account_info.data_len. // The account info might be read only in the vm though, so we translate // to ensure we can write. This is tested by programs/sbf/rust/ro_modify @@ -488,29 +403,18 @@ impl<'a> CallerAccount<'a> { let vm_len_addr = vm_addr .saturating_add(&account_info.data_len as *const u64 as u64) .saturating_sub(account_info as *const _ as *const u64 as u64); - if syscall_parameter_address_restrictions { - // In the same vein as the other check_account_info_pointer() checks, we don't lock - // this pointer to a specific address but we don't want it to be inside accounts, or - // callees might be able to write to the pointed memory. - if vm_len_addr >= solana_sbpf::ebpf::MM_INPUT_START { - return Err(Box::new(CpiError::InvalidPointer)); - } + // In the same vein as the other check_account_info_pointer() checks, we don't lock + // this pointer to a specific address but we don't want it to be inside accounts, or + // callees might be able to write to the pointed memory. + if vm_len_addr >= solana_sbpf::ebpf::MM_INPUT_START { + return Err(Box::new(CpiError::InvalidPointer)); } let ref_to_len_in_vm = translate_type_mut_for_cpi::(memory_mapping, vm_len_addr, false)?; let serialized_data = CallerAccount::get_serialized_data( - memory_mapping, check_aligned, - account_info.data_addr, account_metadata.original_data_len, - if syscall_parameter_address_restrictions { - *ref_to_len_in_vm as usize - } else { - account_info.data_len as usize - }, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, + *ref_to_len_in_vm as usize, )?; Ok(CallerAccount { @@ -550,7 +454,7 @@ pub fn translate_instruction_rust( let check_aligned = invoke_context.get_check_aligned(); let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; let ix = translate_type::(memory_mapping, addr, check_aligned)?; - let account_metas = translate_slice::>( + let account_metas = translate_slice::( memory_mapping, ix.accounts.as_vaddr(), ix.accounts.len(), @@ -569,35 +473,30 @@ pub fn translate_instruction_rust( .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) .unwrap_or(u64::MAX); - // Each account meta is 34 bytes (32 for pubkey, 1 for is_signer, 1 for is_writable) - let account_meta_translation_cost = - (account_metas.len().saturating_mul(size_of::()) as u64) - .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) - .unwrap_or(u64::MAX); + if invoke_context.get_feature_set().increase_cpi_account_info_limit { + // Each account meta is 34 bytes (32 for pubkey, 1 for is_signer, 1 for is_writable) + let account_meta_translation_cost = + (account_metas.len().saturating_mul(size_of::()) as u64) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX); - total_cu_translation_cost = - total_cu_translation_cost.saturating_add(account_meta_translation_cost); + total_cu_translation_cost = + total_cu_translation_cost.saturating_add(account_meta_translation_cost); + } - invoke_context - .compute_meter - .consume_checked(total_cu_translation_cost)?; + consume_compute_meter(invoke_context, total_cu_translation_cost)?; let mut accounts = Vec::with_capacity(account_metas.len()); - for account_meta in account_metas { - // Before using `account_meta` directly, verify that `is_signer` and `is_writable` - // contain valid boolean values to prevent UB. - let account_meta = unsafe { - let ptr = account_meta.as_ptr(); - if (&raw const (*ptr).is_signer).cast::().read_volatile() > 1 - || (&raw const (*ptr).is_writable).cast::().read_volatile() > 1 - { - return Err(Box::new(InstructionError::InvalidArgument)); - } - // SAFETY: VM memory is initialized, and we have validated that the boolean fields - // contain valid data. - account_meta.assume_init_ref() - }; - + #[allow(clippy::needless_range_loop)] + for account_index in 0..account_metas.len() { + #[allow(clippy::indexing_slicing)] + let account_meta = &account_metas[account_index]; + if unsafe { + std::ptr::read_volatile(&account_meta.is_signer as *const _ as *const u8) > 1 + || std::ptr::read_volatile(&account_meta.is_writable as *const _ as *const u8) > 1 + } { + return Err(Box::new(InstructionError::InvalidArgument)); + } accounts.push(account_meta.clone()); } @@ -615,25 +514,32 @@ pub fn translate_accounts_rust<'a>( ) -> Result>, Error> { let check_aligned = invoke_context.get_check_aligned(); let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; - translate_account_infos( + let (account_infos, account_info_keys) = translate_account_infos( account_infos_addr, account_infos_len, |account_info: &AccountInfo| account_info.key as *const _ as u64, + memory_mapping, + invoke_context, + check_aligned, + )?; + + translate_accounts_common( + &account_info_keys, + account_infos, + account_infos_addr, invoke_context, memory_mapping, check_aligned, - |account_infos, account_info_keys| { - translate_accounts_common( - &account_info_keys, - account_infos, - account_infos_addr, + |invoke_context, memory_mapping, check_aligned, _, account_info, account_metadata| { + CallerAccount::from_account_info( invoke_context, memory_mapping, check_aligned, - CallerAccount::from_account_info, + account_info, + account_metadata, ) }, - )? + ) } pub fn translate_signers_rust( @@ -690,7 +596,7 @@ pub fn translate_instruction_c( let ix_c = translate_type::(memory_mapping, addr, check_aligned)?; let program_id = translate_type::(memory_mapping, ix_c.program_id_addr, check_aligned)?; - let account_metas = translate_slice::>( + let account_metas = translate_slice::( memory_mapping, ix_c.accounts_addr, ix_c.accounts_len, @@ -704,35 +610,30 @@ pub fn translate_instruction_c( .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) .unwrap_or(u64::MAX); - // Each account meta is 34 bytes (32 for pubkey, 1 for is_signer, 1 for is_writable) - let account_meta_translation_cost = (ix_c - .accounts_len - .saturating_mul(size_of::() as u64)) - .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) - .unwrap_or(u64::MAX); + if invoke_context.get_feature_set().increase_cpi_account_info_limit { + // Each account meta is 34 bytes (32 for pubkey, 1 for is_signer, 1 for is_writable) + let account_meta_translation_cost = + (ix_c.accounts_len.saturating_mul(size_of::() as u64)) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX); - total_cu_translation_cost = - total_cu_translation_cost.saturating_add(account_meta_translation_cost); + total_cu_translation_cost = + total_cu_translation_cost.saturating_add(account_meta_translation_cost); + } - invoke_context - .compute_meter - .consume_checked(total_cu_translation_cost)?; + consume_compute_meter(invoke_context, total_cu_translation_cost)?; let mut accounts = Vec::with_capacity(ix_c.accounts_len as usize); - for account_meta in account_metas { - // Before using `account_meta` directly, verify that `is_signer` and `is_writable` - // contain valid boolean values to prevent UB. - let account_meta = unsafe { - let ptr = account_meta.as_ptr(); - if (&raw const (*ptr).is_signer).cast::().read_volatile() > 1 - || (&raw const (*ptr).is_writable).cast::().read_volatile() > 1 - { - return Err(Box::new(InstructionError::InvalidArgument)); - } - // SAFETY: VM memory is initialized, and we have validated that the boolean fields - // contain valid data. - account_meta.assume_init_ref() - }; + #[allow(clippy::needless_range_loop)] + for account_index in 0..ix_c.accounts_len as usize { + #[allow(clippy::indexing_slicing)] + let account_meta = &account_metas[account_index]; + if unsafe { + std::ptr::read_volatile(&account_meta.is_signer as *const _ as *const u8) > 1 + || std::ptr::read_volatile(&account_meta.is_writable as *const _ as *const u8) > 1 + } { + return Err(Box::new(InstructionError::InvalidArgument)); + } let pubkey = translate_type::(memory_mapping, account_meta.pubkey_addr, check_aligned)?; accounts.push(AccountMeta { @@ -756,25 +657,24 @@ pub fn translate_accounts_c<'a>( ) -> Result>, Error> { let check_aligned = invoke_context.get_check_aligned(); let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; - translate_account_infos( + let (account_infos, account_info_keys) = translate_account_infos( account_infos_addr, account_infos_len, |account_info: &SolAccountInfo| account_info.key_addr, + memory_mapping, + invoke_context, + check_aligned, + )?; + + translate_accounts_common( + &account_info_keys, + account_infos, + account_infos_addr, invoke_context, memory_mapping, check_aligned, - |account_infos, account_info_keys| { - translate_accounts_common( - &account_info_keys, - account_infos, - account_infos_addr, - invoke_context, - memory_mapping, - check_aligned, - CallerAccount::from_sol_account_info, - ) - }, - )? + CallerAccount::from_sol_account_info, + ) } pub fn translate_signers_c( @@ -835,21 +735,20 @@ pub fn cpi_common( // // Translate the inputs to the syscall and synchronize the caller's account // changes so the callee can see them. - let amount = invoke_context.get_execution_cost().invoke_units; - invoke_context.compute_meter.consume_checked(amount)?; - let syscall_parameter_address_restrictions = invoke_context - .get_feature_set() - .syscall_parameter_address_restrictions; - let virtual_address_space_adjustments = invoke_context - .get_feature_set() - .virtual_address_space_adjustments; - let account_data_direct_mapping = invoke_context.get_feature_set().account_data_direct_mapping; + consume_compute_meter( + invoke_context, + invoke_context.get_execution_cost().invoke_units, + )?; + if let Some(execute_time) = invoke_context.execute_time.as_mut() { + execute_time.stop(); + invoke_context.timings.execute_us += execute_time.as_us(); + } let check_aligned = invoke_context.get_check_aligned(); + let memory_mapping = invoke_context.memory_contexts.memory_mapping_mut()? as *mut MemoryMapping; let instruction = S::translate_instruction(instruction_addr, invoke_context)?; - let instruction_context = invoke_context - .transaction_context - .get_current_instruction_context()?; + let transaction_context = &invoke_context.transaction_context; + let instruction_context = transaction_context.get_current_instruction_context()?; let caller_program_id = instruction_context.get_program_key()?; let signers = S::translate_signers( caller_program_id, @@ -862,37 +761,24 @@ pub fn cpi_common( let mut accounts = S::translate_accounts(account_infos_addr, account_infos_len, invoke_context)?; + let memory_mapping = unsafe { &mut *memory_mapping }; - if syscall_parameter_address_restrictions { - // before initiating CPI, the caller may have modified the - // account (caller_account). We need to update the corresponding - // BorrowedAccount (callee_account) so the callee can see the - // changes. - let transaction_context = &invoke_context.transaction_context; - let instruction_context = transaction_context.get_current_instruction_context()?; - let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; - for translated_account in accounts.iter_mut() { - let callee_account = instruction_context - .try_borrow_instruction_account(translated_account.index_in_caller)?; - // update_callee_account() is moved from translate_accounts_common() - let update_caller = update_callee_account( - memory_mapping, - check_aligned, - &translated_account.caller_account, - callee_account, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, - )?; - translated_account.update_caller_account_region = - translated_account.update_caller_account_info || update_caller; - } + // Before initiating CPI, synchronize caller changes into the account seen + // by the callee. + let transaction_context = &invoke_context.transaction_context; + let instruction_context = transaction_context.get_current_instruction_context()?; + for translated_account in accounts.iter_mut() { + let callee_account = instruction_context + .try_borrow_instruction_account(translated_account.index_in_caller)?; + let update_caller = + update_callee_account(&translated_account.caller_account, callee_account)?; + translated_account.update_caller_account_region = + translated_account.update_caller_account_info || update_caller; } // Process the callee instruction let mut compute_units_consumed = 0; - invoke_context - .process_instruction(&mut compute_units_consumed, &mut ExecuteTimings::default())?; + invoke_context.process_instruction(&mut compute_units_consumed)?; // re-bind to please the borrow checker let transaction_context = &invoke_context.transaction_context; @@ -907,39 +793,28 @@ pub fn cpi_common( if translated_account.update_caller_account_info { update_caller_account( invoke_context, + memory_mapping, check_aligned, &mut translated_account.caller_account, &mut callee_account, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, )?; } } - if virtual_address_space_adjustments { - let memory_mapping = invoke_context.memory_contexts.memory_mapping_mut()?; - for translated_account in accounts.iter() { - let mut callee_account = instruction_context - .try_borrow_instruction_account(translated_account.index_in_caller)?; - if translated_account.update_caller_account_region { - unsafe { - // SAFETY: lifetime is valid by construction: we're resetting the caller memory - // region back to the account that was here before the CPI call, meaning that - // the memory region was guaranteed to be live for sufficient duration upon - // call of this function. - update_caller_account_region( - memory_mapping, - check_aligned, - &translated_account.caller_account, - &mut callee_account, - account_data_direct_mapping, - )?; - } - } + for translated_account in accounts.iter() { + let mut callee_account = instruction_context + .try_borrow_instruction_account(translated_account.index_in_caller)?; + if translated_account.update_caller_account_region { + update_caller_account_region( + memory_mapping, + check_aligned, + &translated_account.caller_account, + &mut callee_account, + )?; } } + invoke_context.execute_time = Some(Measure::start("execute")); Ok(SUCCESS) } @@ -951,26 +826,22 @@ pub struct TranslatedAccount<'a> { pub update_caller_account_info: bool, } -fn translate_account_infos( +fn translate_account_infos<'a, T, F>( account_infos_addr: u64, account_infos_len: u64, - key_addr: impl Fn(&T) -> u64, + key_addr: F, + memory_mapping: &'a MemoryMapping, invoke_context: &InvokeContext, - memory_mapping: &MemoryMapping, check_aligned: bool, - cb: impl FnOnce(&[T], Vec<&Pubkey>) -> R, -) -> Result { - let syscall_parameter_address_restrictions = invoke_context - .get_feature_set() - .syscall_parameter_address_restrictions; - +) -> Result<(&'a [T], Vec<&'a Pubkey>), Error> +where + F: Fn(&T) -> u64, +{ // In the same vein as the other check_account_info_pointer() checks, we don't lock // this pointer to a specific address but we don't want it to be inside accounts, or // callees might be able to write to the pointed memory. - if syscall_parameter_address_restrictions - && account_infos_addr - .saturating_add(account_infos_len.saturating_mul(std::mem::size_of::() as u64)) - >= ebpf::MM_INPUT_START + if account_infos_addr.saturating_add(account_infos_len.saturating_mul(size_of::() as u64)) + >= ebpf::MM_INPUT_START { return Err(CpiError::InvalidPointer.into()); } @@ -981,19 +852,23 @@ fn translate_account_infos( account_infos_len, check_aligned, )?; - check_account_infos(account_infos.len())?; + check_account_infos(account_infos.len(), invoke_context)?; - let account_infos_bytes = account_infos.len().saturating_mul(ACCOUNT_INFO_BYTE_SIZE); + if invoke_context.get_feature_set().increase_cpi_account_info_limit { + let account_infos_bytes = account_infos.len().saturating_mul(ACCOUNT_INFO_BYTE_SIZE); - let amount = (account_infos_bytes as u64) - .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) - .unwrap_or(u64::MAX); - invoke_context.compute_meter.consume_checked(amount)?; + consume_compute_meter( + invoke_context, + (account_infos_bytes as u64) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX), + )?; + } let mut account_info_keys = Vec::with_capacity(account_infos_len as usize); - #[expect(clippy::needless_range_loop)] + #[allow(clippy::needless_range_loop)] for account_index in 0..account_infos_len as usize { - #[expect(clippy::indexing_slicing)] + #[allow(clippy::indexing_slicing)] let account_info = &account_infos[account_index]; account_info_keys.push(translate_type::( memory_mapping, @@ -1001,7 +876,7 @@ fn translate_account_infos( check_aligned, )?); } - Ok(cb(account_infos, account_info_keys)) + Ok((account_infos, account_info_keys)) } // Finish translating accounts and build TranslatedAccount from CallerAccount. @@ -1032,19 +907,7 @@ where // unwrapping here is fine: we're in a syscall and the method below fails // only outside syscalls - let accounts_metadata = &invoke_context - .memory_contexts - .memory_context_abi_v1() - .unwrap() - .accounts_metadata; - - let syscall_parameter_address_restrictions = invoke_context - .get_feature_set() - .syscall_parameter_address_restrictions; - let virtual_address_space_adjustments = invoke_context - .get_feature_set() - .virtual_address_space_adjustments; - let account_data_direct_mapping = invoke_context.get_feature_set().account_data_direct_mapping; + let accounts_metadata = &invoke_context.get_syscall_context().unwrap().accounts_metadata; for (instruction_account_index, instruction_account) in next_instruction_accounts.iter().enumerate() @@ -1063,33 +926,33 @@ where .transaction_context .get_key_of_account_at_index(instruction_account.index_in_transaction)?; - #[expect(deprecated)] + #[allow(deprecated)] if callee_account.is_executable() { // Use the known account - let amount = (callee_account.get_data().len() as u64) - .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) - .unwrap_or(u64::MAX); - invoke_context.compute_meter.consume_checked(amount)?; + consume_compute_meter( + invoke_context, + (callee_account.get_data().len() as u64) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX), + )?; } else if let Some(caller_account_index) = account_info_keys.iter().position(|key| *key == account_key) { let serialized_metadata = - accounts_metadata - .get(index_in_caller as usize) - .ok_or_else(|| { - ic_msg!( - invoke_context, - "Internal error: index mismatch for account {}", - account_key - ); - Box::new(InstructionError::MissingAccount) as Error - })?; + accounts_metadata.get(index_in_caller as usize).ok_or_else(|| { + ic_msg!( + invoke_context, + "Internal error: index mismatch for account {}", + account_key + ); + Box::new(InstructionError::MissingAccount) as Error + })?; // build the CallerAccount corresponding to this account. if caller_account_index >= account_infos.len() { return Err(Box::new(CpiError::InvalidLength)); } - #[expect(clippy::indexing_slicing)] + #[allow(clippy::indexing_slicing)] let caller_account = do_translate( invoke_context, @@ -1102,36 +965,17 @@ where serialized_metadata, )?; - if syscall_parameter_address_restrictions { - // Moved from do_translate() via feature gate. - let amount = (*caller_account.ref_to_len_in_vm) + consume_compute_meter( + invoke_context, + (*caller_account.ref_to_len_in_vm) .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) - .unwrap_or(u64::MAX); - invoke_context.compute_meter.consume_checked(amount)?; - } - let update_caller = if syscall_parameter_address_restrictions { - // update_callee_account() is moved to cpi_common() - true - } else { - // before initiating CPI, the caller may have modified the - // account (caller_account). We need to update the corresponding - // BorrowedAccount (callee_account) so the callee can see the - // changes. - update_callee_account( - memory_mapping, - check_aligned, - &caller_account, - callee_account, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, - )? - }; + .unwrap_or(u64::MAX), + )?; accounts.push(TranslatedAccount { index_in_caller, caller_account, - update_caller_account_region: instruction_account.is_writable() || update_caller, + update_caller_account_region: true, update_caller_account_info: instruction_account.is_writable(), }); } else { @@ -1147,6 +991,11 @@ where Ok(accounts) } +fn consume_compute_meter(invoke_context: &InvokeContext, amount: u64) -> Result<(), Error> { + invoke_context.consume_checked(amount)?; + Ok(()) +} + // Update the given account before executing CPI. // // caller_account and callee_account describe the same account. At CPI entry @@ -1156,16 +1005,11 @@ where // This method updates callee_account so the CPI callee can see the caller's // changes. // -// When true is returned, the caller account must be updated after CPI. This -// is only set for virtual_address_space_adjustments when the pointer may have changed. +// When true is returned, the caller account's mapped data region must be +// updated after CPI because the pointer may have changed. fn update_callee_account( - memory_mapping: &MemoryMapping, - check_aligned: bool, caller_account: &CallerAccount, mut callee_account: BorrowedInstructionAccount<'_, '_>, - syscall_parameter_address_restrictions: bool, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, ) -> Result { let mut must_update_caller = false; @@ -1173,44 +1017,12 @@ fn update_callee_account( callee_account.set_lamports(*caller_account.lamports)?; } - if virtual_address_space_adjustments { - let prev_len = callee_account.get_data().len(); - let post_len = *caller_account.ref_to_len_in_vm as usize; - if prev_len != post_len { - if !account_data_direct_mapping && post_len < prev_len { - // If the account has been shrunk, we're going to zero the unused memory - // *that was previously used*. - let serialized_data = CallerAccount::get_serialized_data( - memory_mapping, - check_aligned, - caller_account.vm_data_addr, - caller_account.original_data_len, - prev_len, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, - )?; - serialized_data - .get_mut(post_len..) - .ok_or_else(|| Box::new(InstructionError::AccountDataTooSmall) as Error)? - .fill(0); - } - callee_account.set_data_length(post_len)?; - // pointer to data may have changed, so caller must be updated - must_update_caller = true; - } - if !account_data_direct_mapping && callee_account.can_data_be_changed().is_ok() { - callee_account.set_data_from_slice(caller_account.serialized_data)?; - } - } else { - // The redundant check helps to avoid the expensive data comparison if we can - match callee_account.can_data_be_resized(caller_account.serialized_data.len()) { - Ok(()) => callee_account.set_data_from_slice(caller_account.serialized_data)?, - Err(err) if callee_account.get_data() != caller_account.serialized_data => { - return Err(Box::new(err)); - } - _ => {} - } + let prev_len = callee_account.get_data().len(); + let post_len = *caller_account.ref_to_len_in_vm as usize; + if prev_len != post_len { + callee_account.set_data_length(post_len)?; + // pointer to data may have changed, so caller must be updated + must_update_caller = true; } // Change the owner at the end so that we are allowed to change the lamports and data before @@ -1223,24 +1035,17 @@ fn update_callee_account( Ok(must_update_caller) } -/// # Safety -/// -/// The the account data pointed to by `callee_account` must outlive the uses of the -/// [`MemoryMapping`]. -unsafe fn update_caller_account_region( +fn update_caller_account_region( memory_mapping: &mut MemoryMapping, check_aligned: bool, caller_account: &CallerAccount, callee_account: &mut BorrowedInstructionAccount<'_, '_>, - account_data_direct_mapping: bool, ) -> Result<(), Error> { let is_caller_loader_deprecated = !check_aligned; let address_space_reserved_for_account = if is_caller_loader_deprecated { caller_account.original_data_len } else { - caller_account - .original_data_len - .saturating_add(MAX_PERMITTED_DATA_INCREASE) + caller_account.original_data_len.saturating_add(MAX_PERMITTED_DATA_INCREASE) }; if address_space_reserved_for_account > 0 { @@ -1251,18 +1056,8 @@ unsafe fn update_caller_account_region( .ok_or_else(|| Box::new(InstructionError::MissingAccount) as Error)?; // vm_data_addr must always point to the beginning of the region debug_assert_eq!(region.vm_addr, caller_account.vm_data_addr); - let mut new_region; - if !account_data_direct_mapping { - new_region = region.clone(); - modify_memory_region_of_account(callee_account, &mut new_region); - } else { - new_region = create_memory_region_of_account(callee_account, region.vm_addr)?; - } + let new_region = create_memory_region_of_account(callee_account, region.vm_addr)?; unsafe { - // SAFETY: the lifetime invariants are delegated to the callers of this function. Both - // `modify_memory_region_of_account` and `create_memory_region_of_account` create memory - // regions pointing to valid buffers by the virtue of the region being produced out of - // an intermediate slice, which itself must be wholly valid. memory_mapping.replace_region(region_index, new_region)?; } } @@ -1279,17 +1074,15 @@ unsafe fn update_caller_account_region( // This method updates caller_account so the CPI caller can see the callee's // changes. // -// Safety: Once `syscall_parameter_address_restrictions` is enabled all fields of [CallerAccount] used -// in this function should never point inside the address space reserved for -// accounts (regardless of the current size of an account). +// Safety: Pointer validation guarantees that all fields of [CallerAccount] +// used here point outside the address space reserved for accounts, regardless +// of an account's current size. fn update_caller_account( invoke_context: &InvokeContext, + memory_mapping: &MemoryMapping, check_aligned: bool, caller_account: &mut CallerAccount<'_>, callee_account: &mut BorrowedInstructionAccount<'_, '_>, - syscall_parameter_address_restrictions: bool, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, ) -> Result<(), Error> { *caller_account.lamports = callee_account.get_lamports(); *caller_account.owner = *callee_account.get_owner(); @@ -1297,18 +1090,13 @@ fn update_caller_account( let prev_len = *caller_account.ref_to_len_in_vm as usize; let post_len = callee_account.get_data().len(); let is_caller_loader_deprecated = !check_aligned; - let address_space_reserved_for_account = - if syscall_parameter_address_restrictions && is_caller_loader_deprecated { - caller_account.original_data_len - } else { - caller_account - .original_data_len - .saturating_add(MAX_PERMITTED_DATA_INCREASE) - }; + let address_space_reserved_for_account = if is_caller_loader_deprecated { + caller_account.original_data_len + } else { + caller_account.original_data_len.saturating_add(MAX_PERMITTED_DATA_INCREASE) + }; - if post_len > address_space_reserved_for_account - && (syscall_parameter_address_restrictions || prev_len != post_len) - { + if post_len > address_space_reserved_for_account { let max_increase = address_space_reserved_for_account.saturating_sub(caller_account.original_data_len); ic_msg!( @@ -1318,59 +1106,19 @@ fn update_caller_account( return Err(Box::new(InstructionError::InvalidRealloc)); } - let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; if prev_len != post_len { - // when virtual_address_space_adjustments is enabled we don't cache the serialized data in - // caller_account.serialized_data. See CallerAccount::from_account_info. - if !(virtual_address_space_adjustments && account_data_direct_mapping) { - // If the account has been shrunk, we're going to zero the unused memory - // *that was previously used*. - if post_len < prev_len { - caller_account - .serialized_data - .get_mut(post_len..) - .ok_or_else(|| Box::new(InstructionError::AccountDataTooSmall) as Error)? - .fill(0); - } - // Set the length of caller_account.serialized_data to post_len. - caller_account.serialized_data = CallerAccount::get_serialized_data( - memory_mapping, - check_aligned, - caller_account.vm_data_addr, - caller_account.original_data_len, - post_len, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, - )?; - } // this is the len field in the AccountInfo::data slice *caller_account.ref_to_len_in_vm = post_len as u64; // this is the len field in the serialized parameters let serialized_len_ptr = translate_type_mut_for_cpi::( memory_mapping, - caller_account - .vm_data_addr - .saturating_sub(std::mem::size_of::() as u64), + caller_account.vm_data_addr.saturating_sub(size_of::() as u64), check_aligned, )?; *serialized_len_ptr = post_len as u64; } - if !(virtual_address_space_adjustments && account_data_direct_mapping) { - // Propagate changes in the callee up to the caller. - let to_slice = &mut caller_account.serialized_data; - let from_slice = callee_account - .get_data() - .get(0..post_len) - .ok_or(CpiError::InvalidLength)?; - if to_slice.len() != from_slice.len() { - return Err(Box::new(InstructionError::AccountDataTooSmall)); - } - to_slice.copy_from_slice(from_slice); - } - Ok(()) } @@ -1381,14 +1129,11 @@ mod tests { use { super::*, crate::{ - invoke_context::BpfAllocator, - memory::translate_type, - memory_context::{MemoryContext, SerializedAccountMetadata}, + invoke_context::BpfAllocator, memory_context::MemoryContext, with_mock_invoke_context_with_feature_set, }, assert_matches::assert_matches, - solana_account::{Account, AccountSharedData, ReadableAccount}, - solana_account_info::AccountInfo, + solana_account::{Account, AccountSharedData}, solana_sbpf::{ ebpf::MM_INPUT_START, memory_region::MemoryRegion, program::SBPFVersion, vm::Config, }, @@ -1398,13 +1143,7 @@ mod tests { IndexOfAccount, instruction_accounts::InstructionAccount, transaction_accounts::KeyedAccountSharedData, }, - std::{ - cell::{Cell, RefCell}, - mem, ptr, - rc::Rc, - slice, - }, - test_case::case, + std::{mem, ptr, slice}, }; macro_rules! mock_invoke_context { @@ -1429,10 +1168,7 @@ mod tests { .into_iter() .map(|a| (a.0, a.1)) .collect::>(); - let mut feature_set = SVMFeatureSet::all_enabled(); - feature_set.syscall_parameter_address_restrictions = false; - feature_set.virtual_address_space_adjustments = false; - feature_set.account_data_direct_mapping = false; + let feature_set = SVMFeatureSet::all_enabled(); let feature_set = &feature_set; with_mock_invoke_context_with_feature_set!( $invoke_context, @@ -1454,20 +1190,13 @@ mod tests { macro_rules! borrow_instruction_account { ($borrowed_account:ident, $invoke_context:expr, $index:expr) => { - let instruction_context = $invoke_context - .transaction_context - .get_current_instruction_context() - .unwrap(); - let $borrowed_account = instruction_context - .try_borrow_instruction_account($index) - .unwrap(); + let instruction_context = + $invoke_context.transaction_context.get_current_instruction_context().unwrap(); + let $borrowed_account = + instruction_context.try_borrow_instruction_account($index).unwrap(); }; } - fn is_zeroed(data: &[u8]) -> bool { - data.iter().all(|b| *b == 0) - } - struct MockCallerAccount { lamports: u64, owner: Pubkey, @@ -1475,53 +1204,22 @@ mod tests { data: Vec, len: u64, regions: Vec, - virtual_address_space_adjustments: bool, } impl MockCallerAccount { - fn new( - lamports: u64, - owner: Pubkey, - data: &[u8], - virtual_address_space_adjustments: bool, - ) -> MockCallerAccount { + fn new(lamports: u64, owner: Pubkey, data: &[u8]) -> MockCallerAccount { let vm_addr = MM_INPUT_START; let mut region_addr = vm_addr; - let region_len = mem::size_of::() - + if virtual_address_space_adjustments { - 0 - } else { - data.len() + MAX_PERMITTED_DATA_INCREASE - }; + let region_len = mem::size_of::(); let mut d = vec![0; region_len]; let mut regions = vec![]; - // always write the [len] part even when virtual_address_space_adjustments unsafe { ptr::write_unaligned::(d.as_mut_ptr().cast(), data.len() as u64) }; - // write the account data when not virtual_address_space_adjustments - if !virtual_address_space_adjustments { - d[mem::size_of::()..][..data.len()].copy_from_slice(data); - } - - // create a region for [len][data+realloc if !virtual_address_space_adjustments] - regions.push(MemoryRegion::new(&raw mut d[..region_len], vm_addr)); + regions.push(MemoryRegion::new(&raw mut d[..], vm_addr)); region_addr += region_len as u64; - if virtual_address_space_adjustments { - // create a region for the directly mapped data - regions.push(MemoryRegion::new(&raw const data[..], region_addr)); - region_addr += data.len() as u64; - - // create a region for the realloc padding - regions.push(MemoryRegion::new( - &raw mut d[mem::size_of::()..], - region_addr, - )); - } else { - // caller_account.serialized_data must have the actual data length - d.truncate(mem::size_of::() + data.len()); - } + regions.push(MemoryRegion::new(&raw const data[..], region_addr)); MockCallerAccount { lamports, @@ -1530,140 +1228,21 @@ mod tests { data: d, len: data.len() as u64, regions, - virtual_address_space_adjustments, - } - } - - fn data_slice<'a>(&self) -> &'a [u8] { - // lifetime crimes - unsafe { - slice::from_raw_parts( - self.data[mem::size_of::()..].as_ptr(), - self.data.capacity() - mem::size_of::(), - ) } } fn caller_account(&mut self) -> CallerAccount<'_> { - let data = if self.virtual_address_space_adjustments { - &mut [] - } else { - &mut self.data[mem::size_of::()..] - }; CallerAccount { lamports: &mut self.lamports, owner: &mut self.owner, original_data_len: self.len as usize, - serialized_data: data, + serialized_data: &mut [], vm_data_addr: self.vm_addr + mem::size_of::() as u64, ref_to_len_in_vm: &mut self.len, } } } - struct MockAccountInfo<'a> { - key: Pubkey, - is_signer: bool, - is_writable: bool, - lamports: u64, - data: &'a [u8], - owner: Pubkey, - executable: bool, - _unused: u64, - } - - impl MockAccountInfo<'_> { - fn new(key: Pubkey, account: &AccountSharedData) -> MockAccountInfo<'_> { - MockAccountInfo { - key, - is_signer: false, - is_writable: false, - lamports: account.lamports(), - data: account.data(), - owner: *account.owner(), - executable: account.executable(), - _unused: account.rent_epoch(), - } - } - - fn into_region(self, vm_addr: u64) -> (Vec, MemoryRegion, SerializedAccountMetadata) { - let size = mem::size_of::() - + mem::size_of::() * 2 - + mem::size_of::>>() - + mem::size_of::() - + mem::size_of::>>() - + self.data.len(); - let mut data = vec![0; size]; - - let vm_addr = vm_addr as usize; - let key_addr = vm_addr + mem::size_of::(); - let lamports_cell_addr = key_addr + mem::size_of::(); - let lamports_addr = lamports_cell_addr + mem::size_of::>>(); - let owner_addr = lamports_addr + mem::size_of::(); - let data_cell_addr = owner_addr + mem::size_of::(); - let data_addr = data_cell_addr + mem::size_of::>>(); - - #[allow(deprecated)] - #[allow(clippy::used_underscore_binding)] - let info = AccountInfo { - key: unsafe { (key_addr as *const Pubkey).as_ref() }.unwrap(), - is_signer: self.is_signer, - is_writable: self.is_writable, - lamports: unsafe { - Rc::from_raw((lamports_cell_addr + RcBox::<&mut u64>::VALUE_OFFSET) as *const _) - }, - data: unsafe { - Rc::from_raw((data_cell_addr + RcBox::<&mut [u8]>::VALUE_OFFSET) as *const _) - }, - owner: unsafe { (owner_addr as *const Pubkey).as_ref() }.unwrap(), - executable: self.executable, - _unused: self._unused, - }; - - unsafe { - ptr::write_unaligned(data.as_mut_ptr().cast(), info); - ptr::write_unaligned( - (data.as_mut_ptr() as usize + key_addr - vm_addr) as *mut _, - self.key, - ); - ptr::write_unaligned( - (data.as_mut_ptr() as usize + lamports_cell_addr - vm_addr) as *mut _, - RcBox::new(RefCell::new((lamports_addr as *mut u64).as_mut().unwrap())), - ); - ptr::write_unaligned( - (data.as_mut_ptr() as usize + lamports_addr - vm_addr) as *mut _, - self.lamports, - ); - ptr::write_unaligned( - (data.as_mut_ptr() as usize + owner_addr - vm_addr) as *mut _, - self.owner, - ); - ptr::write_unaligned( - (data.as_mut_ptr() as usize + data_cell_addr - vm_addr) as *mut _, - RcBox::new(RefCell::new(slice::from_raw_parts_mut( - data_addr as *mut u8, - self.data.len(), - ))), - ); - data[data_addr - vm_addr..].copy_from_slice(self.data); - } - - let region = MemoryRegion::new(&raw mut data[..], vm_addr as u64); - ( - data, - region, - SerializedAccountMetadata { - vm_addr: vm_addr as u64, - original_data_len: self.data.len(), - vm_key_addr: key_addr as u64, - vm_lamports_addr: lamports_addr as u64, - vm_owner_addr: owner_addr as u64, - vm_data_addr: data_addr as u64, - }, - ) - } - } - struct MockInstruction { program_id: Pubkey, accounts: Vec, @@ -1710,24 +1289,6 @@ mod tests { } } - #[repr(C)] - struct RcBox { - strong: Cell, - weak: Cell, - value: T, - } - - impl RcBox { - const VALUE_OFFSET: usize = mem::size_of::>() * 2; - fn new(value: T) -> RcBox { - RcBox { - strong: Cell::new(0), - weak: Cell::new(0), - value, - } - } - } - type TestTransactionAccount = (Pubkey, AccountSharedData, bool); fn transaction_with_one_writable_instruction_account( @@ -1872,10 +1433,15 @@ mod tests { ..Config::default() }; let memory_mapping = - unsafe { MemoryMapping::new(vec![region], &config, SBPFVersion::V3).unwrap() }; + unsafe { MemoryMapping::new(vec![region], &config, SBPFVersion::V3) }.unwrap(); invoke_context .memory_contexts - .mock_set_mapping_abi_v1(memory_mapping); + .set_memory_context_abi_v1(MemoryContext::new( + BpfAllocator::new(solana_program_entrypoint::HEAP_LENGTH as u64), + Vec::new(), + memory_mapping, + )) + .unwrap(); let ins = translate_instruction_rust(vm_addr, &invoke_context).unwrap(); assert_eq!(ins.program_id, program_id); @@ -1902,79 +1468,23 @@ mod tests { let vm_addr = MM_INPUT_START; let (_mem, region) = mock_signers(&[b"foo", &[bump_seed]], vm_addr); - let config = Config { - aligned_memory_mapping: false, - ..Config::default() - }; - let mapping = - unsafe { MemoryMapping::new(vec![region], &config, SBPFVersion::V3).unwrap() }; - invoke_context - .memory_contexts - .set_memory_context_abi_v1(MemoryContext::new( - BpfAllocator::new(0), - Vec::new(), - mapping, - )) - .unwrap(); - - let signers = translate_signers_rust(&program_id, vm_addr, 1, &invoke_context).unwrap(); - assert_eq!(signers[0], derived_key); - } - - #[test] - fn test_translate_accounts_rust() { - let transaction_accounts = - transaction_with_one_writable_instruction_account(b"foobar".to_vec()); - let account = transaction_accounts[1].1.clone(); - let key = transaction_accounts[1].0; - let original_data_len = account.data().len(); - - let vm_addr = MM_INPUT_START; - let (_mem, region, account_metadata) = - MockAccountInfo::new(key, &account).into_region(vm_addr); - let config = Config { aligned_memory_mapping: false, ..Config::default() }; let memory_mapping = - unsafe { MemoryMapping::new(vec![region], &config, SBPFVersion::V3).unwrap() }; - - mock_invoke_context!( - invoke_context, - transaction_context, - b"instruction data", - transaction_accounts, - 0, - &[1, 1] - ); - + unsafe { MemoryMapping::new(vec![region], &config, SBPFVersion::V3) }.unwrap(); invoke_context .memory_contexts .set_memory_context_abi_v1(MemoryContext::new( BpfAllocator::new(solana_program_entrypoint::HEAP_LENGTH as u64), - vec![account_metadata], + Vec::new(), memory_mapping, )) .unwrap(); - invoke_context - .transaction_context - .configure_next_cpi_for_tests( - 0, - vec![ - InstructionAccount::new(1, false, true), - InstructionAccount::new(1, false, true), - ], - vec![], - ) - .unwrap(); - - let accounts = translate_accounts_rust(vm_addr, 1, &invoke_context).unwrap(); - assert_eq!(accounts.len(), 1); - let caller_account = &accounts[0].caller_account; - assert_eq!(caller_account.serialized_data, account.data()); - assert_eq!(caller_account.original_data_len, original_data_len); + let signers = translate_signers_rust(&program_id, vm_addr, 1, &invoke_context).unwrap(); + assert_eq!(signers[0], derived_key); } #[test] @@ -1991,89 +1501,18 @@ mod tests { &[1] ); - let config = Config { - aligned_memory_mapping: false, - ..Config::default() - }; - let memory_mapping = - unsafe { MemoryMapping::new(vec![], &config, SBPFVersion::V3).unwrap() }; - assert_matches!( CallerAccount::get_serialized_data( - &memory_mapping, true, // check_aligned - MM_INPUT_START, account.data().len(), account.data().len().saturating_add(MAX_PERMITTED_DATA_INCREASE).saturating_add(1), - true, // syscall_parameter_address_restrictions - true, // virtual_address_space_adjustments - false, // account_data_direct_mapping ), Err(error) if error.downcast_ref::().unwrap() == &InstructionError::InvalidRealloc ); } #[test] - fn test_caller_account_from_account_info() { - let transaction_accounts = - transaction_with_one_writable_instruction_account(b"foo".to_vec()); - let account = transaction_accounts[1].1.clone(); - mock_invoke_context!( - invoke_context, - transaction_context, - b"instruction data", - transaction_accounts, - 0, - &[1] - ); - - let key = Pubkey::new_unique(); - let vm_addr = MM_INPUT_START; - let (_mem, region, account_metadata) = - MockAccountInfo::new(key, &account).into_region(vm_addr); - - let config = Config { - aligned_memory_mapping: false, - ..Config::default() - }; - let memory_mapping = - unsafe { MemoryMapping::new(vec![region], &config, SBPFVersion::V3).unwrap() }; - - let account_info = translate_type::(&memory_mapping, vm_addr, false).unwrap(); - - invoke_context - .memory_contexts - .mock_set_mapping_abi_v1(memory_mapping); - let check_aligned = invoke_context.get_check_aligned(); - let memory_mapping = invoke_context.memory_contexts.memory_mapping().unwrap(); - let caller_account = CallerAccount::from_account_info( - &invoke_context, - memory_mapping, - check_aligned, - vm_addr, - account_info, - &account_metadata, - ) - .unwrap(); - assert_eq!(*caller_account.lamports, account.lamports()); - assert_eq!(caller_account.owner, account.owner()); - assert_eq!(caller_account.original_data_len, account.data().len()); - assert_eq!( - *caller_account.ref_to_len_in_vm as usize, - account.data().len() - ); - assert_eq!(caller_account.serialized_data, account.data()); - } - - #[case(false, false, false)] - #[case(true, false, false)] - #[case(true, true, false)] - #[case(true, true, true)] - fn test_update_caller_account_lamports_owner( - syscall_parameter_address_restrictions: bool, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, - ) { + fn test_update_caller_account_lamports_owner() { let transaction_accounts = transaction_with_one_writable_instruction_account(vec![]); let account = transaction_accounts[1].1.clone(); mock_invoke_context!( @@ -2086,7 +1525,7 @@ mod tests { ); let mut mock_caller_account = - MockCallerAccount::new(1234, *account.owner(), account.data(), false); + MockCallerAccount::new(1234, *account.owner(), account.data()); let config = Config { aligned_memory_mapping: false, @@ -2098,33 +1537,22 @@ mod tests { &config, SBPFVersion::V3, ) - .unwrap() - }; - invoke_context - .memory_contexts - .mock_set_mapping_abi_v1(memory_mapping); + } + .unwrap(); let mut caller_account = mock_caller_account.caller_account(); - let instruction_context = invoke_context - .transaction_context - .get_current_instruction_context() - .unwrap(); - let mut callee_account = instruction_context - .try_borrow_instruction_account(0) - .unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + let mut callee_account = instruction_context.try_borrow_instruction_account(0).unwrap(); callee_account.set_lamports(42).unwrap(); - callee_account - .set_owner(Pubkey::new_unique().as_ref()) - .unwrap(); + callee_account.set_owner(Pubkey::new_unique().as_ref()).unwrap(); update_caller_account( &invoke_context, + &memory_mapping, true, // check_aligned &mut caller_account, &mut callee_account, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, ) .unwrap(); @@ -2149,7 +1577,7 @@ mod tests { ); let mut mock_caller_account = - MockCallerAccount::new(account.lamports(), *account.owner(), account.data(), false); + MockCallerAccount::new(account.lamports(), *account.owner(), account.data()); let config = Config { aligned_memory_mapping: false, @@ -2161,57 +1589,33 @@ mod tests { &config, SBPFVersion::V3, ) - .unwrap() - }; - invoke_context - .memory_contexts - .mock_set_mapping_abi_v1(memory_mapping); + } + .unwrap(); - let data_slice = mock_caller_account.data_slice(); - let len_ptr = unsafe { - data_slice - .as_ptr() - .offset(-(mem::size_of::() as isize)) - }; + let len_ptr = mock_caller_account.data.as_ptr(); let serialized_len = || unsafe { *len_ptr.cast::() as usize }; let mut caller_account = mock_caller_account.caller_account(); - let instruction_context = invoke_context - .transaction_context - .get_current_instruction_context() - .unwrap(); - let mut callee_account = instruction_context - .try_borrow_instruction_account(0) - .unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + let mut callee_account = instruction_context.try_borrow_instruction_account(0).unwrap(); - for (new_value, expected_realloc_size) in [ - (b"foo".to_vec(), MAX_PERMITTED_DATA_INCREASE + 3), - (b"foobaz".to_vec(), MAX_PERMITTED_DATA_INCREASE), - (b"foobazbad".to_vec(), MAX_PERMITTED_DATA_INCREASE - 3), - ] { - assert_eq!(caller_account.serialized_data, callee_account.get_data()); + for new_value in [b"foo".to_vec(), b"foobaz".to_vec(), b"foobazbad".to_vec()] { + assert!(caller_account.serialized_data.is_empty()); callee_account.set_data_from_slice(&new_value).unwrap(); update_caller_account( &invoke_context, + &memory_mapping, true, // check_aligned &mut caller_account, &mut callee_account, - false, // syscall_parameter_address_restrictions - false, // virtual_address_space_adjustments - false, // account_data_direct_mapping ) .unwrap(); let data_len = callee_account.get_data().len(); assert_eq!(data_len, *caller_account.ref_to_len_in_vm as usize); assert_eq!(data_len, serialized_len()); - assert_eq!(data_len, caller_account.serialized_data.len()); - assert_eq!( - callee_account.get_data(), - &caller_account.serialized_data[..data_len] - ); - assert_eq!(data_slice[data_len..].len(), expected_realloc_size); - assert!(is_zeroed(&data_slice[data_len..])); + assert!(caller_account.serialized_data.is_empty()); } callee_account @@ -2219,17 +1623,14 @@ mod tests { .unwrap(); update_caller_account( &invoke_context, + &memory_mapping, true, // check_aligned &mut caller_account, &mut callee_account, - false, // syscall_parameter_address_restrictions - false, // virtual_address_space_adjustments - false, // account_data_direct_mapping ) .unwrap(); let data_len = callee_account.get_data().len(); - assert_eq!(data_slice[data_len..].len(), 0); - assert!(is_zeroed(&data_slice[data_len..])); + assert_eq!(data_len, serialized_len()); callee_account .set_data_length(original_data_len + MAX_PERMITTED_DATA_INCREASE + 1) @@ -2237,44 +1638,31 @@ mod tests { assert_matches!( update_caller_account( &invoke_context, + &memory_mapping, true, // check_aligned &mut caller_account, &mut callee_account, - false, // syscall_parameter_address_restrictions - false, // virtual_address_space_adjustments - false, // account_data_direct_mapping ), Err(error) if error.downcast_ref::().unwrap() == &InstructionError::InvalidRealloc ); // close the account callee_account.set_data_length(0).unwrap(); - callee_account - .set_owner(system_program::id().as_ref()) - .unwrap(); + callee_account.set_owner(system_program::id().as_ref()).unwrap(); update_caller_account( &invoke_context, + &memory_mapping, true, // check_aligned &mut caller_account, &mut callee_account, - false, // syscall_parameter_address_restrictions - false, // virtual_address_space_adjustments - false, // account_data_direct_mapping ) .unwrap(); let data_len = callee_account.get_data().len(); assert_eq!(data_len, 0); } - #[case(false, false, false)] - #[case(true, false, false)] - #[case(true, true, false)] - #[case(true, true, true)] - fn test_update_callee_account_lamports_owner( - syscall_parameter_address_restrictions: bool, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, - ) { + #[test] + fn test_update_callee_account_lamports_owner() { let transaction_accounts = transaction_with_one_writable_instruction_account(vec![]); let account = transaction_accounts[1].1.clone(); @@ -2288,19 +1676,7 @@ mod tests { ); let mut mock_caller_account = - MockCallerAccount::new(1234, *account.owner(), account.data(), false); - let config = Config { - aligned_memory_mapping: false, - ..Config::default() - }; - let memory_mapping = unsafe { - MemoryMapping::new( - mock_caller_account.regions.clone(), - &config, - SBPFVersion::V3, - ) - .unwrap() - }; + MockCallerAccount::new(1234, *account.owner(), account.data()); let caller_account = mock_caller_account.caller_account(); borrow_instruction_account!(callee_account, invoke_context, 0); @@ -2308,31 +1684,15 @@ mod tests { *caller_account.lamports = 42; *caller_account.owner = Pubkey::new_unique(); - update_callee_account( - &memory_mapping, - true, // check_aligned - &caller_account, - callee_account, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, - ) - .unwrap(); + update_callee_account(&caller_account, callee_account).unwrap(); borrow_instruction_account!(callee_account, invoke_context, 0); assert_eq!(callee_account.get_lamports(), 42); assert_eq!(caller_account.owner, callee_account.get_owner()); } - #[case(false, false, false)] - #[case(true, false, false)] - #[case(true, true, false)] - #[case(true, true, true)] - fn test_update_callee_account_data_writable( - syscall_parameter_address_restrictions: bool, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, - ) { + #[test] + fn test_update_callee_account_data_writable() { let transaction_accounts = transaction_with_one_writable_instruction_account(b"foobar".to_vec()); let account = transaction_accounts[1].1.clone(); @@ -2347,104 +1707,41 @@ mod tests { ); let mut mock_caller_account = - MockCallerAccount::new(1234, *account.owner(), account.data(), false); - let config = Config { - aligned_memory_mapping: false, - ..Config::default() - }; - let memory_mapping = unsafe { - MemoryMapping::new( - mock_caller_account.regions.clone(), - &config, - SBPFVersion::V3, - ) - .unwrap() - }; + MockCallerAccount::new(1234, *account.owner(), account.data()); let mut caller_account = mock_caller_account.caller_account(); borrow_instruction_account!(callee_account, invoke_context, 0); - // Data is not copied in update_callee_account() with virtual_address_space_adjustments - caller_account.serialized_data[0] = b'b'; - update_callee_account( - &memory_mapping, - true, // check_aligned - &caller_account, - callee_account, - false, // syscall_parameter_address_restrictions, - false, // virtual_address_space_adjustments, - false, // account_data_direct_mapping - ) - .unwrap(); - borrow_instruction_account!(callee_account, invoke_context, 0); - assert_eq!(callee_account.get_data(), b"boobar"); + assert!(caller_account.serialized_data.is_empty()); + assert!(!update_callee_account(&caller_account, callee_account).unwrap()); // growing resize - let mut data = b"foobarbaz".to_vec(); - *caller_account.ref_to_len_in_vm = data.len() as u64; - caller_account.serialized_data = &mut data; - assert_eq!( - update_callee_account( - &memory_mapping, - true, // check_aligned - &caller_account, - callee_account, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, - ) - .unwrap(), - virtual_address_space_adjustments, - ); + *caller_account.ref_to_len_in_vm = b"foobarbaz".len() as u64; + borrow_instruction_account!(callee_account, invoke_context, 0); + assert!(update_callee_account(&caller_account, callee_account).unwrap(),); + borrow_instruction_account!(callee_account, invoke_context, 0); + assert_eq!(callee_account.get_data().len(), b"foobarbaz".len()); + drop(callee_account); // truncating resize - let mut data = b"baz".to_vec(); - *caller_account.ref_to_len_in_vm = data.len() as u64; - caller_account.serialized_data = &mut data; + *caller_account.ref_to_len_in_vm = b"baz".len() as u64; borrow_instruction_account!(callee_account, invoke_context, 0); - assert_eq!( - update_callee_account( - &memory_mapping, - true, // check_aligned - &caller_account, - callee_account, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, - ) - .unwrap(), - virtual_address_space_adjustments, - ); + assert!(update_callee_account(&caller_account, callee_account).unwrap(),); + borrow_instruction_account!(callee_account, invoke_context, 0); + assert_eq!(callee_account.get_data().len(), b"baz".len()); + drop(callee_account); // close the account - let mut data = Vec::new(); - caller_account.serialized_data = &mut data; *caller_account.ref_to_len_in_vm = 0; let mut owner = system_program::id(); caller_account.owner = &mut owner; borrow_instruction_account!(callee_account, invoke_context, 0); - update_callee_account( - &memory_mapping, - true, // check_aligned - &caller_account, - callee_account, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, - ) - .unwrap(); + update_callee_account(&caller_account, callee_account).unwrap(); borrow_instruction_account!(callee_account, invoke_context, 0); assert_eq!(callee_account.get_data(), b""); } - #[case(false, false, false)] - #[case(true, false, false)] - #[case(true, true, false)] - #[case(true, true, true)] - fn test_update_callee_account_data_readonly( - syscall_parameter_address_restrictions: bool, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, - ) { + #[test] + fn test_update_callee_account_data_readonly() { let transaction_accounts = transaction_with_one_readonly_instruction_account(b"foobar".to_vec()); let account = transaction_accounts[1].1.clone(); @@ -2459,70 +1756,22 @@ mod tests { ); let mut mock_caller_account = - MockCallerAccount::new(1234, *account.owner(), account.data(), false); - let config = Config { - aligned_memory_mapping: false, - ..Config::default() - }; - let memory_mapping = unsafe { - MemoryMapping::new( - mock_caller_account.regions.clone(), - &config, - SBPFVersion::V3, - ) - .unwrap() - }; - let mut caller_account = mock_caller_account.caller_account(); - borrow_instruction_account!(callee_account, invoke_context, 0); - - // Data is not copied in update_callee_account() with virtual_address_space_adjustments - caller_account.serialized_data[0] = b'b'; - assert_matches!( - update_callee_account( - &memory_mapping, - true, // check_aligned - &caller_account, - callee_account, - false, // syscall_parameter_address_restrictions, - false, // virtual_address_space_adjustments, - false, // account_data_direct_mapping - ), - Err(error) if error.downcast_ref::().unwrap() == &InstructionError::ExternalAccountDataModified - ); + MockCallerAccount::new(1234, *account.owner(), account.data()); + let caller_account = mock_caller_account.caller_account(); // growing resize - let mut data = b"foobarbaz".to_vec(); - *caller_account.ref_to_len_in_vm = data.len() as u64; - caller_account.serialized_data = &mut data; + *caller_account.ref_to_len_in_vm = b"foobarbaz".len() as u64; borrow_instruction_account!(callee_account, invoke_context, 0); assert_matches!( - update_callee_account( - &memory_mapping, - true, // check_aligned - &caller_account, - callee_account, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, - ), + update_callee_account(&caller_account, callee_account), Err(error) if error.downcast_ref::().unwrap() == &InstructionError::AccountDataSizeChanged ); // truncating resize - let mut data = b"baz".to_vec(); - *caller_account.ref_to_len_in_vm = data.len() as u64; - caller_account.serialized_data = &mut data; + *caller_account.ref_to_len_in_vm = b"baz".len() as u64; borrow_instruction_account!(callee_account, invoke_context, 0); assert_matches!( - update_callee_account( - &memory_mapping, - true, // check_aligned - &caller_account, - callee_account, - syscall_parameter_address_restrictions, - virtual_address_space_adjustments, - account_data_direct_mapping, - ), + update_callee_account(&caller_account, callee_account), Err(error) if error.downcast_ref::().unwrap() == &InstructionError::AccountDataSizeChanged ); } diff --git a/solana/program-runtime/src/deploy.rs b/solana/program-runtime/src/deploy.rs index e1853a8..58d522f 100644 --- a/solana/program-runtime/src/deploy.rs +++ b/solana/program-runtime/src/deploy.rs @@ -1,19 +1,15 @@ //! Program deployment functionality. -#[cfg(feature = "metrics")] -use {crate::program_metrics::LoadProgramMetrics, solana_svm_measure::measure::Measure}; use { crate::{ invoke_context::InvokeContext, - loaded_programs::{ProgramCacheForTxBatch, ProgramRuntimeEnvironment}, - program_cache_entry::{DELAY_VISIBILITY_SLOT_OFFSET, ProgramCacheEntry}, + loaded_programs::{ProgramCacheEntry, ProgramCacheForTxBatch, ProgramRuntimeEnvironment}, }, - solana_clock::Slot, solana_instruction::error::InstructionError, solana_pubkey::Pubkey, solana_sbpf::{ elf::{ElfError, Executable}, - program::{BuiltinProgram, SBPFVersion}, + program::BuiltinProgram, verifier::RequisiteVerifier, }, solana_svm_log_collector::{LogCollector, ic_logger_msg}, @@ -21,19 +17,19 @@ use { std::{cell::RefCell, rc::Rc}, }; -fn morph_into_deployment_environment( - from: ProgramRuntimeEnvironment, - disable_sbpf_v0_v1_v2_deployment: bool, -) -> Result>, ElfError> { - let mut config = (*from).get_config().clone(); +fn morph_into_deployment_environment_v1<'a>( + from: Arc>>, +) -> Result>, ElfError> { + let mut config = from.get_config().clone(); config.reject_broken_elfs = true; - if disable_sbpf_v0_v1_v2_deployment { - config.enabled_sbpf_versions = SBPFVersion::V3..=*config.enabled_sbpf_versions.end(); - } + // Once the tests are being build using a toolchain which supports the newer SBPF versions, + // the deployment of older versions will be disabled: + // config.enabled_sbpf_versions = + // *config.enabled_sbpf_versions.end()..=*config.enabled_sbpf_versions.end(); let mut result = BuiltinProgram::new_loader(config); - for (_key, (name, value)) in (*from).get_function_registry().iter() { + for (_key, (name, value)) in from.get_function_registry().iter() { // Deployment of programs with sol_alloc_free is disabled. So do not register the syscall. if name != *b"sol_alloc_free_" { result.register_function(unsafe { std::str::from_utf8_unchecked(name) }, value)?; @@ -49,34 +45,19 @@ fn morph_into_deployment_environment( #[allow(clippy::too_many_arguments)] pub fn deploy_program( log_collector: Option>>, - #[cfg(feature = "metrics")] load_program_metrics: &mut LoadProgramMetrics, program_cache_for_tx_batch: &mut ProgramCacheForTxBatch, program_runtime_environment: ProgramRuntimeEnvironment, - disable_sbpf_v0_v1_v2_deployment: bool, program_id: &Pubkey, - loader_key: &Pubkey, - account_size: usize, programdata: &[u8], - deployment_slot: Slot, ) -> Result<(), InstructionError> { - #[cfg(feature = "metrics")] - let mut register_syscalls_time = Measure::start("register_syscalls_time"); - let deployment_program_runtime_environment = morph_into_deployment_environment( - ProgramRuntimeEnvironment::clone(&program_runtime_environment), - disable_sbpf_v0_v1_v2_deployment, - ) + let deployment_program_runtime_environment = morph_into_deployment_environment_v1(Arc::clone( + &*program_runtime_environment, + )) .map_err(|e| { ic_logger_msg!(log_collector, "Failed to register syscalls: {}", e); InstructionError::ProgramEnvironmentSetupFailure })?; - #[cfg(feature = "metrics")] - { - register_syscalls_time.stop(); - load_program_metrics.register_syscalls_us = register_syscalls_time.as_us(); - } // Verify using stricter deployment_program_runtime_environment - #[cfg(feature = "metrics")] - let mut load_elf_time = Measure::start("load_elf_time"); let executable = Executable::::load( programdata, Arc::new(deployment_program_runtime_environment), @@ -85,75 +66,41 @@ pub fn deploy_program( ic_logger_msg!(log_collector, "{}", err); InstructionError::InvalidAccountData })?; - #[cfg(feature = "metrics")] - { - load_elf_time.stop(); - load_program_metrics.load_elf_us = load_elf_time.as_us(); - } - #[cfg(feature = "metrics")] - let mut verify_code_time = Measure::start("verify_code_time"); executable.verify::().map_err(|err| { ic_logger_msg!(log_collector, "{}", err); InstructionError::InvalidAccountData })?; - #[cfg(feature = "metrics")] - { - verify_code_time.stop(); - load_program_metrics.verify_code_us = verify_code_time.as_us(); - } // Reload but with program_runtime_environment let executor = unsafe { // SAFETY: The executable has been verified just above. - ProgramCacheEntry::reload( - loader_key, - program_runtime_environment, - deployment_slot, - deployment_slot.saturating_add(DELAY_VISIBILITY_SLOT_OFFSET), - programdata, - account_size, - #[cfg(feature = "metrics")] - load_program_metrics, - ) + ProgramCacheEntry::reload(program_runtime_environment, programdata) } .map_err(|err| { ic_logger_msg!(log_collector, "{}", err); InstructionError::InvalidAccountData })?; - if let Some(old_entry) = program_cache_for_tx_batch.find(program_id) { - executor.stats.merge_from(&old_entry.stats); - } - #[cfg(feature = "metrics")] - { - load_program_metrics.program_id = program_id.to_string(); - } program_cache_for_tx_batch.store_modified_entry(*program_id, Arc::new(executor)); Ok(()) } #[macro_export] macro_rules! deploy_program { - ($invoke_context:expr, - $program_id:expr, - $loader_key:expr, - $account_size:expr, - $programdata:expr, - $deployment_slot:expr, - $disable_sbpf_v0_v1_v2_deployment:expr $(,)?) => { + ($invoke_context:expr, $program_id:expr, $loader_key:expr, $account_size:expr, $programdata:expr, $deployment_slot:expr $(,)?) => { assert_eq!( $deployment_slot, $invoke_context.program_cache_for_tx_batch.slot() ); #[cfg(feature = "metrics")] - let mut load_program_metrics = $crate::program_metrics::LoadProgramMetrics::default(); + let mut load_program_metrics = $crate::loaded_programs::LoadProgramMetrics::default(); $crate::deploy::deploy_program( $invoke_context.get_log_collector(), #[cfg(feature = "metrics")] &mut load_program_metrics, $invoke_context.program_cache_for_tx_batch, $invoke_context - .get_program_runtime_environment_for_deployment() + .get_program_runtime_environments_for_deployment() + .get_env_for_deployment() .clone(), - $disable_sbpf_v0_v1_v2_deployment, $program_id, $loader_key, $account_size, diff --git a/solana/program-runtime/src/execution_budget.rs b/solana/program-runtime/src/execution_budget.rs index df3d543..3a4b944 100644 --- a/solana/program-runtime/src/execution_budget.rs +++ b/solana/program-runtime/src/execution_budget.rs @@ -17,26 +17,41 @@ fn get_max_instruction_stack_depth(simd_0268_active: bool) -> usize { } } -//Default CPI invocation cost -pub const DEFAULT_INVOCATION_COST: u64 = 946; +/// Default CPI invocation cost. +pub const DEFAULT_INVOCATION_COST: u64 = 1000; +/// CPI invocation cost with SIMD-0339 active. +pub const INVOKE_UNITS_COST_SIMD_0339: u64 = 946; + +fn get_invoke_unit_cost(simd_0339_active: bool) -> u64 { + if simd_0339_active { + INVOKE_UNITS_COST_SIMD_0339 + } else { + DEFAULT_INVOCATION_COST + } +} /// Max call depth. This is the maximum nesting of SBF to SBF call that can happen within a program. pub const MAX_CALL_DEPTH: usize = 64; +/// The size of one SBF stack frame. +pub const STACK_FRAME_SIZE: usize = 4096; + +/// Maximum compute units a transaction may request. pub const MAX_COMPUTE_UNIT_LIMIT: u32 = 1_400_000; /// Roughly 0.5us/page, where page is 32K; given roughly 15CU/us, the /// default heap page cost = 0.5 * 15 ~= 8CU/page pub const DEFAULT_HEAP_COST: u64 = 8; +/// Default compute-unit limit for an instruction. pub const DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT: u32 = 200_000; -// SIMD-170 defines max CUs to be allocated for any builtin program instructions, that -// have not been migrated to sBPF programs. +/// Maximum compute units allocated to native builtins that have not moved to SBF. pub const MAX_BUILTIN_ALLOCATION_COMPUTE_UNIT_LIMIT: u32 = 3_000; +/// Maximum heap frame size a transaction may request. pub const MAX_HEAP_FRAME_BYTES: u32 = 256 * 1024; +/// Minimum heap frame size. pub const MIN_HEAP_FRAME_BYTES: u32 = HEAP_LENGTH as u32; -/// The total accounts data a transaction can load is limited to 64MiB to not break -/// anyone in Mainnet-beta today. It can be set by set_loaded_accounts_data_size_limit instruction +/// Default loaded account data limit for one transaction. pub const MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES: NonZeroU32 = NonZeroU32::new(64 * 1024 * 1024).unwrap(); @@ -63,7 +78,6 @@ pub struct SVMTransactionExecutionBudget { pub heap_size: u32, } -#[cfg(feature = "dev-context-only-utils")] impl Default for SVMTransactionExecutionBudget { fn default() -> Self { Self::new_with_defaults(/* simd_0268_active */ false) @@ -71,6 +85,7 @@ impl Default for SVMTransactionExecutionBudget { } impl SVMTransactionExecutionBudget { + /// Creates the default execution budget for the selected feature state. pub fn new_with_defaults(simd_0268_active: bool) -> Self { SVMTransactionExecutionBudget { compute_unit_limit: u64::from(MAX_COMPUTE_UNIT_LIMIT), @@ -78,7 +93,7 @@ impl SVMTransactionExecutionBudget { max_instruction_trace_length: MAX_INSTRUCTION_TRACE_LENGTH, sha256_max_slices: 20_000, max_call_depth: MAX_CALL_DEPTH, - stack_frame_size: solana_sbpf::vm::get_stack_frame_size(), + stack_frame_size: STACK_FRAME_SIZE, heap_size: u32::try_from(solana_program_entrypoint::HEAP_LENGTH).unwrap(), } } @@ -204,10 +219,16 @@ pub struct SVMTransactionExecutionCost { impl Default for SVMTransactionExecutionCost { fn default() -> Self { + Self::new_with_defaults(/* simd_0339_active */ false) + } +} + +impl SVMTransactionExecutionCost { + pub fn new_with_defaults(simd_0339_active: bool) -> Self { SVMTransactionExecutionCost { log_64_units: 100, create_program_address_units: 1500, - invoke_units: DEFAULT_INVOCATION_COST, + invoke_units: get_invoke_unit_cost(simd_0339_active), sha256_base_cost: 85, sha256_byte_cost: 1, log_pubkey_units: 100, @@ -258,9 +279,7 @@ impl Default for SVMTransactionExecutionCost { bls12_381_additional_pair_cost: 13_023, } } -} -impl SVMTransactionExecutionCost { /// Returns cost of the Poseidon hash function for the given number of /// inputs is determined by the following quadratic function: /// @@ -282,19 +301,21 @@ impl SVMTransactionExecutionCost { /// [0] https://github.com/Lightprotocol/light-poseidon#performance pub fn poseidon_cost(&self, nr_inputs: u64) -> Option { let squared_inputs = nr_inputs.checked_pow(2)?; - let mul_result = self - .poseidon_cost_coefficient_a - .checked_mul(squared_inputs)?; + let mul_result = self.poseidon_cost_coefficient_a.checked_mul(squared_inputs)?; let final_result = mul_result.checked_add(self.poseidon_cost_coefficient_c)?; Some(final_result) } } +/// Execution budget, loaded-data limit, and fee metadata for one transaction. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SVMTransactionExecutionAndFeeBudgetLimits { + /// Compute and invocation limits. pub budget: SVMTransactionExecutionBudget, + /// Maximum loaded account data size. pub loaded_accounts_data_size_limit: u32, + /// Fee metadata carried with the transaction. pub fee_details: FeeDetails, } @@ -311,6 +332,7 @@ impl Default for SVMTransactionExecutionAndFeeBudgetLimits { #[cfg(feature = "dev-context-only-utils")] impl SVMTransactionExecutionAndFeeBudgetLimits { + /// Creates default limits with explicit fee metadata. pub fn with_fee(fee_details: FeeDetails) -> Self { Self { fee_details, diff --git a/solana/program-runtime/src/invoke_context.rs b/solana/program-runtime/src/invoke_context.rs index c81225c..39c82e1 100644 --- a/solana/program-runtime/src/invoke_context.rs +++ b/solana/program-runtime/src/invoke_context.rs @@ -1,63 +1,58 @@ -#[cfg(feature = "dev-context-only-utils")] -use { - crate::program_cache_entry::ProgramCacheEntry, - solana_account::{AccountSharedData, WritableAccount, create_account_shared_data_for_test}, - solana_epoch_schedule::EpochSchedule, - solana_instruction::AccountMeta, - solana_message::{LegacyMessage, Message, SanitizedMessage}, - solana_sdk_ids::sysvar, - solana_transaction_context::transaction_accounts::KeyedAccountSharedData, - std::collections::{HashMap, HashSet}, -}; use { crate::{ execution_budget::{SVMTransactionExecutionBudget, SVMTransactionExecutionCost}, loaded_programs::{ - ProgramCacheForTxBatch, ProgramRuntimeEnvironment, ProgramRuntimeEnvironments, + ProgramCacheEntry, ProgramCacheEntryType, ProgramCacheForTxBatch, + ProgramRuntimeEnvironments, }, memory_context::{MemoryContext, MemoryContexts}, - program_cache_entry::ProgramCacheEntryType, stable_log, sysvar_cache::SysvarCache, }, + solana_account::{AccountSharedData, create_account_shared_data_for_test}, + solana_epoch_schedule::EpochSchedule, solana_hash::Hash, - solana_instruction::{Instruction, error::InstructionError}, + solana_instruction::{AccountMeta, Instruction, error::InstructionError}, solana_pubkey::Pubkey, solana_sbpf::{ ebpf::MM_HEAP_START, - elf::{ElfError, Executable as GenericExecutable}, + elf::Executable as GenericExecutable, error::{EbpfError, ProgramResult}, memory_region::MemoryMapping, - program::{BuiltinProgram, SBPFVersion}, + program::{BuiltinCodegen, BuiltinFunction, SBPFVersion}, vm::{Config, ContextObject, EbpfVm}, }, solana_sdk_ids::{ - bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable, loader_v4, native_loader, + bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable, loader_v4, native_loader, sysvar, }, solana_svm_callback::InvokeContextCallback, solana_svm_feature_set::SVMFeatureSet, solana_svm_log_collector::{LogCollector, ic_msg}, solana_svm_measure::measure::Measure, - solana_svm_timings::{ExecuteDetailsTimings, ExecuteTimings}, - solana_svm_transaction::svm_message::SVMMessage, + solana_svm_timings::ExecuteDetailsTimings, + solana_svm_transaction::{instruction::SVMInstruction, svm_message::SVMMessage}, solana_svm_type_overrides::sync::Arc, solana_transaction_context::{ IndexOfAccount, MAX_ACCOUNTS_PER_TRANSACTION, instruction::InstructionContext, instruction_accounts::InstructionAccount, transaction::TransactionContext, + transaction_accounts::KeyedAccountSharedData, }, std::{ alloc::Layout, borrow::Cow, cell::{Cell, RefCell}, fmt::{self, Debug}, - ptr, + ptr::NonNull, rc::Rc, - time::Duration, }, }; -pub type BuiltinFunctionRegisterer = - fn(&mut BuiltinProgram>, &str) -> Result<(), ElfError>; +pub use crate::memory_context::SerializedAccountMetadata; + +pub type BuiltinFunctionWithContext = ( + BuiltinFunction>, + BuiltinCodegen>, +); pub type Executable = GenericExecutable>; pub type RegisterTrace<'a> = &'a [[u64; 12]]; @@ -76,13 +71,13 @@ macro_rules! declare_process_instruction { _arg4: u64, ) -> Result> { fn process_instruction_inner( - $invoke_context: &mut $crate::invoke_context::InvokeContext, + $invoke_context: &mut $crate::invoke_context::InvokeContext<'_, '_>, ) -> std::result::Result<(), $crate::__private::InstructionError> $inner let consumption_result = if $cu_to_consume > 0 { - invoke_context.compute_meter.consume_checked($cu_to_consume) + invoke_context.consume_checked($cu_to_consume) } else { Ok(()) }; @@ -103,21 +98,19 @@ impl ContextObject for InvokeContext<'_, '_> { // 1 to 1 instruction to compute unit mapping // ignore overflow, Ebpf will bail if exceeded let compute_meter = self.compute_meter.0.get(); - self.compute_meter - .0 - .set(compute_meter.saturating_sub(amount)); + self.compute_meter.0.set(compute_meter.saturating_sub(amount)); } fn get_remaining(&self) -> u64 { self.compute_meter.0.get() } - fn active_mapping_ptr(&mut self) -> ptr::NonNull { - let memory = self + fn active_mapping_ptr(&mut self) -> NonNull { + let memory_mapping = self .memory_contexts .memory_mapping_mut() - .expect("The memory context must have been set for the current instruction"); - ptr::NonNull::from_mut(memory) + .expect("memory context must be set for the current instruction"); + NonNull::from(memory_mapping) } } @@ -141,11 +134,7 @@ impl BpfAllocator { pub fn alloc(&mut self, layout: Layout) -> Result { let bytes_to_align = (self.pos as *const u8).align_offset(layout.align()) as u64; - if self - .pos - .saturating_add(bytes_to_align) - .saturating_add(layout.size() as u64) - <= self.len + if self.pos.saturating_add(bytes_to_align).saturating_add(layout.size() as u64) <= self.len { self.pos = self.pos.saturating_add(bytes_to_align); let addr = MM_HEAP_START.saturating_add(self.pos); @@ -160,43 +149,45 @@ impl BpfAllocator { pub struct EnvironmentConfig<'a> { pub blockhash: Hash, pub blockhash_lamports_per_signature: u64, - alpenglow_migration_succeeded: bool, epoch_stake_callback: &'a dyn InvokeContextCallback, feature_set: &'a SVMFeatureSet, - program_runtime_environments: &'a ProgramRuntimeEnvironments, + pub program_runtime_environments_for_execution: &'a ProgramRuntimeEnvironments, sysvar_cache: &'a SysvarCache, } impl<'a> EnvironmentConfig<'a> { pub fn new( blockhash: Hash, blockhash_lamports_per_signature: u64, - alpenglow_migration_succeeded: bool, epoch_stake_callback: &'a dyn InvokeContextCallback, feature_set: &'a SVMFeatureSet, - program_runtime_environments: &'a ProgramRuntimeEnvironments, + program_runtime_environments_for_execution: &'a ProgramRuntimeEnvironments, sysvar_cache: &'a SysvarCache, ) -> Self { Self { blockhash, blockhash_lamports_per_signature, - alpenglow_migration_succeeded, epoch_stake_callback, feature_set, - program_runtime_environments, + program_runtime_environments_for_execution, sysvar_cache, } } - /// Get cached sysvars + /// Get cached sysvars. pub fn sysvar_cache(&self) -> &SysvarCache { self.sysvar_cache } } +pub struct SyscallContext { + pub allocator: BpfAllocator, + pub accounts_metadata: Vec, +} + pub struct ComputeMeter(Cell); impl ComputeMeter { - /// Consume compute units + /// Consume compute units. pub fn consume_checked(&self, amount: u64) -> Result<(), Box> { let compute_meter = self.0.get(); let exceeded = compute_meter < amount; @@ -207,10 +198,7 @@ impl ComputeMeter { Ok(()) } - /// Set compute units - /// - /// Only use for tests and benchmarks - #[cfg(feature = "dev-context-only-utils")] + /// Set compute units. Only use for tests and benchmarks. pub fn mock_set_remaining(&self, remaining: u64) { self.0.set(remaining); } @@ -232,18 +220,17 @@ pub struct InvokeContext<'a, 'ix_data> { /// the designated compute budget during program execution. pub compute_meter: ComputeMeter, log_collector: Option>>, - /// Time spent so far executing nested program calls. - pub total_nested_exec_time: Duration, + /// Latest measurement not yet accumulated in [ExecuteDetailsTimings::execute_us] + pub execute_time: Option, pub timings: ExecuteDetailsTimings, + pub syscall_context: Vec>, pub memory_contexts: MemoryContexts, /// Pairs of index in TX instruction trace and VM register trace register_traces: Vec<(usize, Vec<[u64; 12]>)>, - /// Debug port to use for this executing transaction. - #[cfg(feature = "sbpf-debugger")] - pub debug_port: Option, } impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { + #[allow(clippy::too_many_arguments)] pub fn new( transaction_context: &'a mut TransactionContext<'ix_data>, program_cache_for_tx_batch: &'a mut ProgramCacheForTxBatch, @@ -260,12 +247,11 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { compute_budget, execution_cost, compute_meter: ComputeMeter(Cell::new(compute_budget.compute_unit_limit)), - total_nested_exec_time: Duration::ZERO, + execute_time: None, timings: ExecuteDetailsTimings::default(), + syscall_context: Vec::new(), memory_contexts: MemoryContexts::new(), register_traces: Vec::new(), - #[cfg(feature = "sbpf-debugger")] - debug_port: None, } } @@ -296,12 +282,14 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { } } + self.syscall_context.push(None); self.memory_contexts.push_placeholder(); self.transaction_context.push() } /// Pop a stack frame from the invocation stack - pub fn pop(&mut self) -> Result<(), InstructionError> { + pub(crate) fn pop(&mut self) -> Result<(), InstructionError> { + self.syscall_context.pop(); self.memory_contexts.pop(); self.transaction_context.pop() } @@ -323,10 +311,8 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { instruction: Instruction, signer_seeds: &[&[&[u8]]], ) -> Result<(), InstructionError> { - let caller_program_id = *self - .transaction_context - .get_current_instruction_context()? - .get_program_key()?; + let caller_program_id = + *self.transaction_context.get_current_instruction_context()?.get_program_key()?; // The conversion from `PubkeyError` to `InstructionError` through // num-traits is incorrect, but it's the existing behavior. let signers = signer_seeds @@ -335,8 +321,20 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { .collect::, solana_pubkey::PubkeyError>>() .map_err(|e| e as u64)?; self.prepare_next_cpi_instruction(instruction, &signers)?; - let mut compute_units_consumed = 0; - self.process_instruction(&mut compute_units_consumed, &mut ExecuteTimings::default())?; + self.process_instruction(&mut 0)?; + Ok(()) + } + + /// Deprecated entrypoint for a cross-program invocation from a builtin program + // NOTE: + // we only keep it around for one special case of CPI with post-delegation actions + pub fn native_invoke( + &mut self, + instruction: Instruction, + signers: &[Pubkey], + ) -> Result<(), InstructionError> { + self.prepare_next_cpi_instruction(instruction, signers)?; + self.process_instruction(&mut 0)?; Ok(()) } @@ -372,9 +370,8 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { })?; debug_assert!((index_in_transaction as usize) < transaction_callee_map.len()); - let index_in_callee = transaction_callee_map - .get_mut(index_in_transaction as usize) - .unwrap(); + let index_in_callee = + transaction_callee_map.get_mut(index_in_transaction as usize).unwrap(); if (*index_in_callee as usize) < instruction_accounts.len() { let cloned_account = { @@ -454,9 +451,8 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { // Find and validate executables / program accounts let callee_program_id = &instruction.program_id; - let program_account_index_in_transaction = self - .transaction_context - .find_index_of_account(callee_program_id); + let program_account_index_in_transaction = + self.transaction_context.find_index_of_account(callee_program_id); let program_account_index_in_instruction = program_account_index_in_transaction .map(|index| instruction_context.get_index_of_account_in_instruction(index)); @@ -488,46 +484,46 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { Ok(()) } - /// Prepare the instruction trace with all the top level instructions - pub fn prepare_top_level_instructions( + /// Helper to prepare for process_instruction()/process_precompile() when the instruction is + /// a top level one + pub fn prepare_next_top_level_instruction( &mut self, - message: &'ix_data impl SVMMessage, - ) -> Result<(), (u8, InstructionError)> { - for (top_level_instruction_index, (_, instruction)) in - message.program_instructions_iter().enumerate() - { - let mut transaction_callee_map: Vec = vec![u16::MAX; MAX_ACCOUNTS_PER_TRANSACTION]; + message: &impl SVMMessage, + instruction: &SVMInstruction, + program_account_index: IndexOfAccount, + data: &'ix_data [u8], + ) -> Result<(), InstructionError> { + // We reference accounts by an u8 index, so we have a total of 256 accounts. + let mut transaction_callee_map: Vec = vec![u16::MAX; MAX_ACCOUNTS_PER_TRANSACTION]; - let mut instruction_accounts: Vec = - Vec::with_capacity(instruction.accounts.len()); - for index_in_transaction in instruction.accounts.iter() { - let index_in_callee = transaction_callee_map - .get_mut(*index_in_transaction as usize) - .expect("Invalid index in transaction"); + let mut instruction_accounts: Vec = + Vec::with_capacity(instruction.accounts.len()); + for index_in_transaction in instruction.accounts.iter() { + debug_assert!((*index_in_transaction as usize) < transaction_callee_map.len()); - if (*index_in_callee as usize) > instruction_accounts.len() { - *index_in_callee = instruction_accounts.len() as u16; - } + let index_in_callee = + transaction_callee_map.get_mut(*index_in_transaction as usize).unwrap(); - let index_in_transaction = *index_in_transaction as usize; - instruction_accounts.push(InstructionAccount::new( - index_in_transaction as IndexOfAccount, - message.is_signer(index_in_transaction), - message.is_writable(index_in_transaction), - )); + if (*index_in_callee as usize) > instruction_accounts.len() { + *index_in_callee = instruction_accounts.len() as u16; } - self.transaction_context - .configure_instruction_at_index( - top_level_instruction_index, - instruction.program_id_index as u16, - instruction_accounts, - transaction_callee_map, - Cow::Borrowed(instruction.data), - None, - ) - .map_err(|err| (top_level_instruction_index as u8, err))?; + let index_in_transaction = *index_in_transaction as usize; + instruction_accounts.push(InstructionAccount::new( + index_in_transaction as IndexOfAccount, + message.is_signer(index_in_transaction), + message.is_writable(index_in_transaction), + )); } + + self.transaction_context.configure_instruction_at_index( + self.transaction_context.get_instruction_trace_length(), + program_account_index, + instruction_accounts, + transaction_callee_map, + Cow::Borrowed(data), + None, + )?; Ok(()) } @@ -535,11 +531,10 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { pub fn process_instruction( &mut self, compute_units_consumed: &mut u64, - timings: &mut ExecuteTimings, ) -> Result<(), InstructionError> { *compute_units_consumed = 0; self.push()?; - self.process_executable_chain(compute_units_consumed, timings) + self.process_executable_chain(compute_units_consumed) // MUST pop if and only if `push` succeeded, independent of `result`. // Thus, the `.and()` instead of an `.and_then()`. .and(self.pop()) @@ -565,104 +560,120 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { fn process_executable_chain( &mut self, compute_units_consumed: &mut u64, - timings: &mut ExecuteTimings, ) -> Result<(), InstructionError> { let instruction_context = self.transaction_context.get_current_instruction_context()?; + let program_id = *instruction_context.get_program_key()?; + let owner_id = instruction_context.get_program_owner()?; let process_executable_chain_time = Measure::start("process_executable_chain_time"); - let builtin_id = { - let owner_id = instruction_context.get_program_owner()?; - if native_loader::check_id(&owner_id) { - *instruction_context.get_program_key()? - } else if bpf_loader_deprecated::check_id(&owner_id) - || bpf_loader::check_id(&owner_id) - || bpf_loader_upgradeable::check_id(&owner_id) - || loader_v4::check_id(&owner_id) - { - owner_id - } else { - return Err(InstructionError::UnsupportedProgramId); - } + let cache_id = if native_loader::check_id(&owner_id) + || bpf_loader_deprecated::check_id(&owner_id) + || bpf_loader::check_id(&owner_id) + || bpf_loader_upgradeable::check_id(&owner_id) + || loader_v4::check_id(&owner_id) + { + program_id + } else { + return Err(InstructionError::UnsupportedProgramId); }; - // The Murmur3 hash value (used by RBPF) of the string "entrypoint" - const ENTRYPOINT_KEY: u32 = 0x71E3CF81; + let pre_remaining_units = self.get_remaining(); let entry = self .program_cache_for_tx_batch - .find(&builtin_id) + .find(&cache_id) .ok_or(InstructionError::UnsupportedProgramId)?; - let function = match &entry.program { - ProgramCacheEntryType::Builtin(program) => program - .get_function_registry() - .lookup_by_key(ENTRYPOINT_KEY) - .map(|(_name, (function, _codegen))| function), - _ => None, - } - .ok_or(InstructionError::UnsupportedProgramId)?; - - let program_id = *instruction_context.get_program_key()?; - self.transaction_context - .set_return_data(program_id, Vec::new())?; - let logger = self.get_log_collector(); - stable_log::program_invoke(&logger, &program_id, self.get_stack_height()); - let pre_remaining_units = self.get_remaining(); - // For now, only built-ins are invoked from here, so the VM and its Config are irrelevant. - self.memory_contexts - .set_memory_context_abi_v1(MemoryContext::new( - BpfAllocator::new(0), - Vec::new(), - // SAFETY: - // This path invokes a builtin program, so this mapping is never used. - unsafe { - MemoryMapping::new(Vec::new(), &Config::default(), SBPFVersion::Reserved) - .unwrap() - }, - ))?; - let mut vm = EbpfVm::new( - Arc::clone( - &**self - .environment_config - .program_runtime_environments - .get_env_for_execution(), - ), - SBPFVersion::V0, - // Removes lifetime tracking - unsafe { std::mem::transmute::<&mut InvokeContext, &mut InvokeContext>(self) }, - 0, - ); - vm.invoke_function(function); - let result = match vm.program_result { - ProgramResult::Ok(_) => { - stable_log::program_success(&logger, &program_id); - Ok(()) - } - ProgramResult::Err(ref err) => { - if let EbpfError::SyscallError(syscall_error) = err { - if let Some(instruction_err) = syscall_error.downcast_ref::() - { - stable_log::program_failure(&logger, &program_id, instruction_err); - Err(instruction_err.clone()) - } else { - stable_log::program_failure(&logger, &program_id, syscall_error); - Err(InstructionError::ProgramFailedToComplete) + let result = match &entry.program { + ProgramCacheEntryType::Builtin(program) => { + // The Murmur3 hash value (used by RBPF) of the string "entrypoint". + const ENTRYPOINT_KEY: u32 = 0x71E3CF81; + let function = program + .get_function_registry() + .lookup_by_key(ENTRYPOINT_KEY) + .map(|(_name, (function, _codegen))| function) + .ok_or(InstructionError::UnsupportedProgramId)?; + + self.transaction_context.set_return_data(program_id, Vec::new())?; + let logger = self.get_log_collector(); + stable_log::program_invoke(&logger, &program_id, self.get_stack_height()); + self.set_syscall_context(SyscallContext { + allocator: BpfAllocator::new(0), + accounts_metadata: Vec::new(), + })?; + self.memory_contexts.set_memory_context_abi_v1(MemoryContext::new( + BpfAllocator::new(0), + Vec::new(), + // Built-ins do not dereference VM memory, but the invoke + // context still requires an active mapping. + unsafe { + MemoryMapping::new(Vec::new(), &Config::default(), SBPFVersion::Reserved) + .unwrap() + }, + ))?; + let mut vm = EbpfVm::new( + Arc::clone( + &**self + .environment_config + .program_runtime_environments_for_execution + .get_env_for_execution(), + ), + SBPFVersion::V0, + // Removes lifetime tracking. + unsafe { std::mem::transmute::<&mut InvokeContext, &mut InvokeContext>(self) }, + 0, + ); + vm.invoke_function(function); + match vm.program_result { + ProgramResult::Ok(_) => { + stable_log::program_success(&logger, &program_id); + Ok(()) + } + ProgramResult::Err(ref err) => { + if let EbpfError::SyscallError(syscall_error) = err { + if let Some(instruction_err) = + syscall_error.downcast_ref::() + { + stable_log::program_failure(&logger, &program_id, instruction_err); + Err(instruction_err.clone()) + } else { + stable_log::program_failure(&logger, &program_id, syscall_error); + Err(InstructionError::ProgramFailedToComplete) + } + } else { + stable_log::program_failure(&logger, &program_id, err); + Err(InstructionError::ProgramFailedToComplete) + } } - } else { - stable_log::program_failure(&logger, &program_id, err); - Err(InstructionError::ProgramFailedToComplete) } } + ProgramCacheEntryType::Loaded(program) => { + self.transaction_context.set_return_data(program_id, Vec::new())?; + let logger = self.get_log_collector(); + stable_log::program_invoke(&logger, &program_id, self.get_stack_height()); + let result = crate::vm::execute(program, self).map_err(|error| { + error + .downcast_ref::() + .cloned() + .unwrap_or(InstructionError::ProgramFailedToComplete) + }); + match &result { + Ok(()) => stable_log::program_success(&logger, &program_id), + Err(error) => stable_log::program_failure(&logger, &program_id, error), + } + result + } + _ => Err(InstructionError::UnsupportedProgramId), }; let post_remaining_units = self.get_remaining(); *compute_units_consumed = pre_remaining_units.saturating_sub(post_remaining_units); - if builtin_id == program_id && result.is_ok() && *compute_units_consumed == 0 { + if matches!(&entry.program, ProgramCacheEntryType::Builtin(_)) + && result.is_ok() + && *compute_units_consumed == 0 + { return Err(InstructionError::BuiltinProgramsMustConsumeComputeUnits); } - timings - .execute_accessories - .process_instructions - .process_executable_chain_us += process_executable_chain_time.end_as_us(); + process_executable_chain_time.end_as_us(); result } @@ -671,9 +682,16 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { self.log_collector.clone() } - #[cfg(feature = "dev-context-only-utils")] - pub fn set_alpenglow_migration_succeeded_for_tests(&mut self, succeeded: bool) { - self.environment_config.alpenglow_migration_succeeded = succeeded; + /// Consume compute units + pub fn consume_checked(&self, amount: u64) -> Result<(), Box> { + self.compute_meter.consume_checked(amount) + } + + /// Set compute units + /// + /// Only use for tests and benchmarks + pub fn mock_set_remaining(&self, remaining: u64) { + self.compute_meter.mock_set_remaining(remaining); } /// Get this invocation's compute budget @@ -691,27 +709,18 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { self.environment_config.feature_set } - pub fn get_program_runtime_environment_for_deployment(&self) -> &ProgramRuntimeEnvironment { - self.environment_config - .program_runtime_environments - .get_env_for_deployment() - } - pub fn is_deprecate_legacy_vote_ixs_active(&self) -> bool { - self.environment_config - .feature_set - .deprecate_legacy_vote_ixs + self.environment_config.feature_set.deprecate_legacy_vote_ixs } - pub fn is_alpenglow_migration_succeeded(&self) -> bool { - self.environment_config.alpenglow_migration_succeeded + /// Get cached sysvars + pub fn get_sysvar_cache(&self) -> &SysvarCache { + self.environment_config.sysvar_cache } /// Get cached epoch total stake. pub fn get_epoch_stake(&self) -> u64 { - self.environment_config - .epoch_stake_callback - .get_epoch_stake() + self.environment_config.epoch_stake_callback.get_epoch_stake() } /// Get cached stake for the epoch vote account. @@ -722,9 +731,7 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { } pub fn is_precompile(&self, pubkey: &Pubkey) -> bool { - self.environment_config - .epoch_stake_callback - .is_precompile(pubkey) + self.environment_config.epoch_stake_callback.is_precompile(pubkey) } // Should alignment be enforced during user pointer translation @@ -740,6 +747,32 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { .unwrap_or(true) } + // Set this instruction syscall context + pub fn set_syscall_context( + &mut self, + syscall_context: SyscallContext, + ) -> Result<(), InstructionError> { + *self.syscall_context.last_mut().ok_or(InstructionError::CallDepth)? = + Some(syscall_context); + Ok(()) + } + + // Get this instruction's SyscallContext + pub fn get_syscall_context(&self) -> Result<&SyscallContext, InstructionError> { + self.syscall_context + .last() + .and_then(std::option::Option::as_ref) + .ok_or(InstructionError::CallDepth) + } + + // Get this instruction's SyscallContext + pub fn get_syscall_context_mut(&mut self) -> Result<&mut SyscallContext, InstructionError> { + self.syscall_context + .last_mut() + .and_then(|syscall_context| syscall_context.as_mut()) + .ok_or(InstructionError::CallDepth) + } + /// Insert a VM register trace pub fn insert_register_trace(&mut self, register_trace: Vec<[u64; 12]>) { if register_trace.is_empty() { @@ -779,7 +812,6 @@ impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { } } -#[cfg(feature = "dev-context-only-utils")] #[macro_export] macro_rules! with_mock_invoke_context_with_feature_set { ( @@ -787,8 +819,7 @@ macro_rules! with_mock_invoke_context_with_feature_set { $transaction_context:ident, $feature_set:ident, $top_level_instructions:literal, - $transaction_accounts:expr, - $all_accounts:expr $(,)? + $transaction_accounts:expr $(,)? ) => { use { solana_svm_callback::InvokeContextCallback, @@ -808,14 +839,6 @@ macro_rules! with_mock_invoke_context_with_feature_set { let compute_budget = SVMTransactionExecutionBudget::new_with_defaults( $feature_set.raise_cpi_nesting_limit_to_8, ); - let mut sysvar_cache = SysvarCache::default(); - sysvar_cache.fill_missing_entries(|pubkey, callback| { - for (key, account) in $all_accounts.iter() { - if key == pubkey { - callback(account.data()); - } - } - }); let mut $transaction_context = TransactionContext::new( $transaction_accounts, Rent::default(), @@ -823,11 +846,18 @@ macro_rules! with_mock_invoke_context_with_feature_set { compute_budget.max_instruction_trace_length, $top_level_instructions, ); - let program_runtime_environments = ProgramRuntimeEnvironments::mock(); + let mut sysvar_cache = SysvarCache::default(); + sysvar_cache.fill_missing_entries(|pubkey, callback| { + for index in 0..$transaction_context.get_number_of_accounts() { + if $transaction_context.get_key_of_account_at_index(index).unwrap() == pubkey { + callback($transaction_context.accounts().try_borrow(index).unwrap().data()); + } + } + }); + let program_runtime_environments = ProgramRuntimeEnvironments::default(); let environment_config = EnvironmentConfig::new( Hash::default(), 0, - false, &MockInvokeContextCallback {}, $feature_set, &program_runtime_environments, @@ -840,25 +870,9 @@ macro_rules! with_mock_invoke_context_with_feature_set { environment_config, Some(LogCollector::new_ref()), compute_budget, - SVMTransactionExecutionCost::default(), - ); - }; - ( - $invoke_context:ident, - $transaction_context:ident, - $feature_set:ident, - $top_level_instructions:literal, - $transaction_accounts:expr $(,)? - ) => { - let transaction_accounts: Vec<(solana_pubkey::Pubkey, solana_account::AccountSharedData)> = - $transaction_accounts; - $crate::with_mock_invoke_context_with_feature_set!( - $invoke_context, - $transaction_context, - $feature_set, - $top_level_instructions, - transaction_accounts, - &transaction_accounts + SVMTransactionExecutionCost::new_with_defaults( + $feature_set.increase_cpi_account_info_limit, + ), ); }; ( @@ -867,7 +881,7 @@ macro_rules! with_mock_invoke_context_with_feature_set { $feature_set:ident, $transaction_accounts:expr $(,)? ) => { - $crate::with_mock_invoke_context_with_feature_set!( + with_mock_invoke_context_with_feature_set!( $invoke_context, $transaction_context, $feature_set, @@ -877,7 +891,6 @@ macro_rules! with_mock_invoke_context_with_feature_set { }; } -#[cfg(feature = "dev-context-only-utils")] #[macro_export] macro_rules! with_mock_invoke_context { ( @@ -909,154 +922,112 @@ macro_rules! with_mock_invoke_context { }; } -#[cfg(feature = "dev-context-only-utils")] -pub fn mock_compile_message( - instruction: &Instruction, - accounts: &[(Pubkey, A)], - program_id: &Pubkey, - loader_key: &Pubkey, -) -> Option<(SanitizedMessage, Vec<(Pubkey, AccountSharedData)>)> -where - AccountSharedData: From, - A: Clone, -{ - let message = Message::new(std::slice::from_ref(instruction), None); - let transaction_accounts: Vec<_> = message - .account_keys - .iter() - .map(|key| { - let account = accounts - .iter() - .find(|(k, _)| k == key) - .map(|(_, a)| AccountSharedData::from(a.clone())) - .unwrap_or_else(|| { - if key == program_id { - let mut account = AccountSharedData::new(0, 0, loader_key); - account.set_executable(true); - account - } else { - AccountSharedData::default() - } - }); - (*key, account) - }) - .collect(); - - let sanitized_message = SanitizedMessage::Legacy(LegacyMessage::new(message, &HashSet::new())); - - Some((sanitized_message, transaction_accounts)) -} - -#[cfg(feature = "dev-context-only-utils")] +#[allow(clippy::too_many_arguments)] pub fn mock_process_instruction_with_feature_set< F: FnMut(&mut InvokeContext), G: FnMut(&mut InvokeContext), >( - program_id: &Pubkey, + loader_id: &Pubkey, + program_index: Option, instruction_data: &[u8], - mut accounts: Vec, + mut transaction_accounts: Vec, instruction_account_metas: Vec, expected_result: Result<(), InstructionError>, - builtin: BuiltinFunctionRegisterer, + builtin_function: BuiltinFunctionWithContext, mut pre_adjustments: F, mut post_adjustments: G, feature_set: &SVMFeatureSet, ) -> Vec { - let original_len = accounts.len(); - if !accounts - .iter() - .any(|(key, _)| *key == sysvar::epoch_schedule::id()) - { - accounts.push(( - sysvar::epoch_schedule::id(), - create_account_shared_data_for_test(&EpochSchedule::default()), + let mut instruction_accounts: Vec = + Vec::with_capacity(instruction_account_metas.len()); + for account_meta in instruction_account_metas.iter() { + let index_in_transaction = transaction_accounts + .iter() + .position(|(key, _account)| *key == account_meta.pubkey) + .unwrap_or(transaction_accounts.len()) + as IndexOfAccount; + instruction_accounts.push(InstructionAccount::new( + index_in_transaction, + account_meta.is_signer, + account_meta.is_writable, )); } - let instruction = - Instruction::new_with_bytes(*program_id, instruction_data, instruction_account_metas); - let (sanitized_message, transaction_accounts) = - mock_compile_message(&instruction, &accounts, program_id, &native_loader::id()).unwrap(); - - let program_owner = accounts - .iter() - .find(|(key, _)| key == program_id) - .map(|(_, acct)| *acct.owner()) - .unwrap_or_else(native_loader::id); - let is_builtin = native_loader::check_id(&program_owner); - + let program_index = if let Some(index) = program_index { + index + } else { + let processor_account = AccountSharedData::new(0, 0, &native_loader::id()); + transaction_accounts.push((*loader_id, processor_account)); + transaction_accounts.len().saturating_sub(1) as IndexOfAccount + }; + let pop_epoch_schedule_account = + if !transaction_accounts.iter().any(|(key, _)| *key == sysvar::epoch_schedule::id()) { + transaction_accounts.push(( + sysvar::epoch_schedule::id(), + create_account_shared_data_for_test(&EpochSchedule::default()), + )); + true + } else { + false + }; with_mock_invoke_context_with_feature_set!( invoke_context, transaction_context, feature_set, - 1, - transaction_accounts, - &accounts + transaction_accounts ); - let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); program_cache_for_tx_batch.replenish( - if is_builtin { - *program_id - } else { - program_owner - }, - Arc::new(ProgramCacheEntry::new_builtin(0, 0, builtin)), + *loader_id, + Arc::new(ProgramCacheEntry::new_builtin(builtin_function)), ); program_cache_for_tx_batch.set_slot_for_tests( invoke_context - .environment_config - .sysvar_cache() + .get_sysvar_cache() .get_clock() .map(|clock| clock.slot) .unwrap_or(1), ); invoke_context.program_cache_for_tx_batch = &mut program_cache_for_tx_batch; - pre_adjustments(&mut invoke_context); - invoke_context - .prepare_top_level_instructions(&sanitized_message) + .transaction_context + .configure_top_level_instruction_for_tests( + program_index, + instruction_accounts, + instruction_data.to_vec(), + ) .unwrap(); - - let result = invoke_context.process_instruction(&mut 0, &mut ExecuteTimings::default()); + let result = invoke_context.process_instruction(&mut 0); assert_eq!(result, expected_result); post_adjustments(&mut invoke_context); - - let txn_result_keys: Vec<_> = (0..transaction_context.get_number_of_accounts()) - .map(|i| *transaction_context.get_key_of_account_at_index(i).unwrap()) - .collect(); - let txn_result_accounts = transaction_context.deconstruct_without_keys().unwrap(); - let txn_result_map = txn_result_keys - .into_iter() - .zip(txn_result_accounts) - .collect::>(); - - accounts - .into_iter() - .take(original_len) - .map(|(key, original)| txn_result_map.get(&key).cloned().unwrap_or(original)) - .collect() + let mut transaction_accounts = transaction_context.deconstruct_without_keys().unwrap(); + if pop_epoch_schedule_account { + transaction_accounts.pop(); + } + transaction_accounts.pop(); + transaction_accounts } -#[cfg(feature = "dev-context-only-utils")] pub fn mock_process_instruction( - program_id: &Pubkey, + loader_id: &Pubkey, + program_index: Option, instruction_data: &[u8], - accounts: Vec, + transaction_accounts: Vec, instruction_account_metas: Vec, expected_result: Result<(), InstructionError>, - builtin: BuiltinFunctionRegisterer, + builtin_function: BuiltinFunctionWithContext, pre_adjustments: F, post_adjustments: G, ) -> Vec { mock_process_instruction_with_feature_set( - program_id, + loader_id, + program_index, instruction_data, - accounts, + transaction_accounts, instruction_account_metas, expected_result, - builtin, + builtin_function, pre_adjustments, post_adjustments, &SVMFeatureSet::all_enabled(), @@ -1067,20 +1038,18 @@ pub fn mock_process_instruction>(); assert_eq!( program_id, - instruction_context - .try_borrow_instruction_account(0)? - .get_owner() + instruction_context.try_borrow_instruction_account(0)?.get_owner() ); assert_ne!( - instruction_context - .try_borrow_instruction_account(1)? - .get_owner(), + instruction_context.try_borrow_instruction_account(1)?.get_owner(), instruction_context.get_key_of_instruction_account(0)? ); @@ -1186,7 +1151,6 @@ mod tests { desired_result, } => { invoke_context - .compute_meter .consume_checked(compute_units_to_consume) .map_err(|_| InstructionError::ComputationalBudgetExceeded)?; return desired_result; @@ -1205,32 +1169,18 @@ mod tests { #[test_case(false; "SIMD-0268 disabled")] #[test_case(true; "SIMD-0268 enabled")] fn test_instruction_stack_height(simd_0268_active: bool) { - let feature_set = &SVMFeatureSet { - raise_cpi_nesting_limit_to_8: simd_0268_active, - ..SVMFeatureSet::all_enabled() - }; - let max_depth = SVMTransactionExecutionBudget::new_with_defaults(simd_0268_active) - .max_instruction_stack_depth; - assert_eq!( - max_depth, - if simd_0268_active { - MAX_INSTRUCTION_STACK_DEPTH_SIMD_0268 - } else { - MAX_INSTRUCTION_STACK_DEPTH - }, - ); - - // Set up max_depth + 1 accounts (one extra to trigger the failing push) - // and a matching program account for each. + let one_more_than_max_depth = + SVMTransactionExecutionBudget::new_with_defaults(simd_0268_active) + .max_instruction_stack_depth + .saturating_add(1); let mut invoke_stack = vec![]; let mut transaction_accounts = vec![]; let mut instruction_accounts = vec![]; - for index in 0..max_depth.saturating_add(1) { - let program_id = solana_pubkey::new_rand(); - invoke_stack.push(program_id); + for index in 0..one_more_than_max_depth { + invoke_stack.push(solana_pubkey::new_rand()); transaction_accounts.push(( solana_pubkey::new_rand(), - AccountSharedData::new(1, 1, &program_id), + AccountSharedData::new(index as u64, 1, invoke_stack.get(index).unwrap()), )); instruction_accounts.push(InstructionAccount::new( index as IndexOfAccount, @@ -1238,10 +1188,6 @@ mod tests { true, )); } - - // Append program accounts after the regular accounts so that - // `first_program_account + depth` indexes the right program. - let first_program_account = transaction_accounts.len(); for (index, program_id) in invoke_stack.iter().enumerate() { transaction_accounts.push(( *program_id, @@ -1253,44 +1199,26 @@ mod tests { false, )); } - with_mock_invoke_context_with_feature_set!( - invoke_context, - transaction_context, - feature_set, - transaction_accounts, - ); + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); - // Each push must succeed and the stack height must track. - for depth in 0..max_depth { - assert_eq!(invoke_context.get_stack_height(), depth); + // Check call depth increases and has a limit + let mut depth_reached: usize = 0; + for _ in 0..invoke_stack.len() { invoke_context .transaction_context .configure_top_level_instruction_for_tests( - (first_program_account.saturating_add(depth)) as IndexOfAccount, + one_more_than_max_depth.saturating_add(depth_reached) as IndexOfAccount, instruction_accounts.clone(), vec![], ) .unwrap(); - assert!( - invoke_context.push().is_ok(), - "push at depth {depth} should succeed (max_depth={max_depth})", - ); + if Err(InstructionError::CallDepth) == invoke_context.push() { + break; + } + depth_reached = depth_reached.saturating_add(1); } - - // At exactly max_depth, one more push must fail with CallDepth. - assert_eq!(invoke_context.get_stack_height(), max_depth); - invoke_context - .transaction_context - .configure_top_level_instruction_for_tests( - (first_program_account.saturating_add(max_depth)) as IndexOfAccount, - instruction_accounts.clone(), - vec![], - ) - .unwrap(); - assert_eq!(invoke_context.push(), Err(InstructionError::CallDepth),); - - // Stack height must not have changed after the rejected push. - assert_eq!(invoke_context.get_stack_height(), max_depth); + assert_ne!(depth_reached, 0); + assert!(depth_reached < one_more_than_max_depth); } #[test] @@ -1338,27 +1266,28 @@ mod tests { 2, ); - transaction_context - .configure_instruction_at_index( - 0, - 0, - vec![InstructionAccount::new(0, false, false)], - vec![u16::MAX; 256], - Cow::Owned(Vec::new()), - None, - ) - .unwrap(); - - transaction_context - .configure_instruction_at_index( - 1, - 0, - vec![InstructionAccount::new(0, false, false)], - vec![u16::MAX; 256], - Cow::Owned(Vec::new()), - None, - ) - .unwrap(); + // To be uncommented when we reorder the trace + // transaction_context + // .configure_instruction_at_index( + // 0, + // 0, + // vec![InstructionAccount::new(0, false, false)], + // vec![u16::MAX; 256], + // Cow::Owned(Vec::new()), + // None, + // ) + // .unwrap(); + // + // transaction_context + // .configure_instruction_at_index( + // 1, + // 0, + // vec![InstructionAccount::new(0, false, false)], + // vec![u16::MAX; 256], + // Cow::Owned(Vec::new()), + // None, + // ) + // .unwrap(); for _ in 0..MAX_INSTRUCTIONS { transaction_context.push().unwrap(); @@ -1420,7 +1349,10 @@ mod tests { let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); program_cache_for_tx_batch.replenish( callee_program_id, - Arc::new(ProgramCacheEntry::new_builtin(0, 1, MockBuiltin::register)), + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), ); invoke_context.program_cache_for_tx_batch = &mut program_cache_for_tx_batch; @@ -1475,7 +1407,10 @@ mod tests { let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); program_cache_for_tx_batch.replenish( callee_program_id, - Arc::new(ProgramCacheEntry::new_builtin(0, 1, MockBuiltin::register)), + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), ); invoke_context.program_cache_for_tx_batch = &mut program_cache_for_tx_batch; @@ -1494,13 +1429,10 @@ mod tests { }, metas, ); - invoke_context - .prepare_next_cpi_instruction(inner_instruction, &[]) - .unwrap(); + invoke_context.prepare_next_cpi_instruction(inner_instruction, &[]).unwrap(); let mut compute_units_consumed = 0; - let result = invoke_context - .process_instruction(&mut compute_units_consumed, &mut ExecuteTimings::default()); + let result = invoke_context.process_instruction(&mut compute_units_consumed); // Because the instruction had compute cost > 0, then regardless of the execution result, // the number of compute units consumed should be a non-default which is something greater @@ -1559,7 +1491,10 @@ mod tests { let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); program_cache_for_tx_batch.replenish( program_key, - Arc::new(ProgramCacheEntry::new_builtin(0, 0, MockBuiltin::register)), + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), ); invoke_context.program_cache_for_tx_batch = &mut program_cache_for_tx_batch; @@ -1570,7 +1505,7 @@ mod tests { .transaction_context .configure_top_level_instruction_for_tests(2, instruction_accounts, instruction_data) .unwrap(); - let result = invoke_context.process_instruction(&mut 0, &mut ExecuteTimings::default()); + let result = invoke_context.process_instruction(&mut 0); assert!(result.is_ok()); assert_eq!( @@ -1608,10 +1543,8 @@ mod tests { .push((key, AccountSharedData::new(1, 1, &Pubkey::new_unique()))); account_metas.push(AccountMeta::new_readonly(key, false)); } else { - let repeated_key = transaction_accounts - .get(i % MAX_ACCOUNTS_PER_TRANSACTION) - .unwrap() - .0; + let repeated_key = + transaction_accounts.get(i % MAX_ACCOUNTS_PER_TRANSACTION).unwrap().0; account_metas.push(AccountMeta::new_readonly(repeated_key, false)); } } @@ -1636,10 +1569,8 @@ mod tests { .unwrap(); fn test_case_1(invoke_context: &InvokeContext) { - let instruction_context = invoke_context - .transaction_context - .get_next_instruction_context() - .unwrap(); + let instruction_context = + invoke_context.transaction_context.get_next_instruction_context().unwrap(); for index_in_instruction in 0..MAX_ACCOUNTS_REFERENCED as IndexOfAccount { let index_in_transaction = instruction_context .get_index_of_instruction_account_in_transaction(index_in_instruction) @@ -1664,10 +1595,8 @@ mod tests { } fn test_case_2(invoke_context: &InvokeContext) { - let instruction_context = invoke_context - .transaction_context - .get_next_instruction_context() - .unwrap(); + let instruction_context = + invoke_context.transaction_context.get_next_instruction_context().unwrap(); for index_in_instruction in 0..MAX_ACCOUNTS_REFERENCED as IndexOfAccount { let index_in_transaction = instruction_context .get_index_of_instruction_account_in_transaction(index_in_instruction) @@ -1694,14 +1623,30 @@ mod tests { } } + let svm_instruction = + SVMInstruction::from(sanitized.message().instructions().first().unwrap()); invoke_context - .prepare_top_level_instructions(&sanitized) + .prepare_next_top_level_instruction( + &sanitized, + &svm_instruction, + 90, + svm_instruction.data, + ) .unwrap(); test_case_1(&invoke_context); invoke_context.transaction_context.push().unwrap(); - invoke_context.transaction_context.pop().unwrap(); + let svm_instruction = + SVMInstruction::from(sanitized.message().instructions().get(1).unwrap()); + invoke_context + .prepare_next_top_level_instruction( + &sanitized, + &svm_instruction, + 90, + svm_instruction.data, + ) + .unwrap(); test_case_2(&invoke_context); @@ -1760,16 +1705,21 @@ mod tests { let sanitized = SanitizedTransaction::try_from_legacy_transaction(transaction, &HashSet::new()) .unwrap(); + let svm_instruction = + SVMInstruction::from(sanitized.message().instructions().first().unwrap()); invoke_context - .prepare_top_level_instructions(&sanitized) + .prepare_next_top_level_instruction( + &sanitized, + &svm_instruction, + 90, + svm_instruction.data, + ) .unwrap(); { - let instruction_context = invoke_context - .transaction_context - .get_next_instruction_context() - .unwrap(); + let instruction_context = + invoke_context.transaction_context.get_next_instruction_context().unwrap(); for index_in_instruction in 2..account_metas.len() as IndexOfAccount { let is_duplicate = instruction_context .is_instruction_account_duplicate(index_in_instruction) @@ -1793,10 +1743,8 @@ mod tests { invoke_context .prepare_next_cpi_instruction(instruction, &[fee_payer.pubkey()]) .unwrap(); - let instruction_context = invoke_context - .transaction_context - .get_next_instruction_context() - .unwrap(); + let instruction_context = + invoke_context.transaction_context.get_next_instruction_context().unwrap(); for index_in_instruction in 2..account_metas.len().saturating_sub(1) as u16 { let is_duplicate = instruction_context .is_instruction_account_duplicate(index_in_instruction) @@ -1847,7 +1795,10 @@ mod tests { let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); program_cache_for_tx_batch.replenish( TEST_CALLEE_PROGRAM_ID, - Arc::new(ProgramCacheEntry::new_builtin(0, 1, MockBuiltin::register)), + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), ); invoke_context.program_cache_for_tx_batch = &mut program_cache_for_tx_batch; @@ -1955,36 +1906,4 @@ mod tests { "top-level signer should not need seeds: {result:?}" ); } - - #[test] - fn test_compile_message() { - let program_id = Pubkey::new_from_array([1u8; 32]); - let writable = Pubkey::new_from_array([2u8; 32]); - let loader_key = Pubkey::new_from_array([3u8; 32]); - - let instruction = Instruction { - program_id, - accounts: vec![AccountMeta::new(writable, false)], - data: vec![1, 2, 3], - }; - - let accounts = vec![( - writable, - Account { - lamports: 100, - ..Account::default() - }, - )]; - - let (message, tx_accounts) = - mock_compile_message(&instruction, &accounts, &program_id, &loader_key).unwrap(); - - assert_eq!(message.instructions().len(), 1); - assert_eq!(tx_accounts.len(), 2); - assert_eq!(tx_accounts.first().unwrap().0, writable); - assert_eq!(tx_accounts.get(1).unwrap().0, program_id); - - // Verify the writable account is NOT promoted to signer. - assert!(!message.is_signer(0)); - } } diff --git a/solana/program-runtime/src/lib.rs b/solana/program-runtime/src/lib.rs index 4bc7e25..de3c845 100644 --- a/solana/program-runtime/src/lib.rs +++ b/solana/program-runtime/src/lib.rs @@ -1,6 +1,7 @@ -#![cfg(feature = "agave-unstable-api")] +#![allow(clippy::disallowed_methods)] #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::indexing_slicing)] +#![doc = include_str!("../README.md")] pub use solana_sbpf; pub mod cpi; @@ -8,12 +9,9 @@ pub mod deploy; pub mod execution_budget; pub mod invoke_context; pub mod loaded_programs; -pub mod loading_task; pub mod mem_pool; pub mod memory; pub mod memory_context; -pub mod program_cache_entry; -pub mod program_metrics; pub mod serialization; pub mod stable_log; pub mod sysvar_cache; diff --git a/solana/program-runtime/src/loaded_programs.rs b/solana/program-runtime/src/loaded_programs.rs index 183f98c..38e0939 100644 --- a/solana/program-runtime/src/loaded_programs.rs +++ b/solana/program-runtime/src/loaded_programs.rs @@ -1,258 +1,225 @@ use { - crate::{ - invoke_context::InvokeContext, - loading_task::LoadingTaskWaiter, - program_cache_entry::{ProgramCacheEntry, ProgramCacheEntryType, retention_score}, - program_metrics::{EMA_SCALE, ProgramCacheStats}, - }, - log::error, - percentage::PercentageInteger, - solana_clock::{Epoch, Slot}, + crate::invoke_context::{BuiltinFunctionWithContext, InvokeContext}, + solana_clock::Slot, solana_pubkey::Pubkey, - solana_sbpf::program::BuiltinProgram, - solana_svm_type_overrides::{ - rand::{Rng, rng}, - sync::{Arc, Mutex, RwLock, atomic::Ordering}, - thread, + solana_sbpf::{ + elf::Executable, program::BuiltinProgram, verifier::RequisiteVerifier, vm::Config, }, + solana_svm_type_overrides::sync::Arc, std::{ - collections::{HashMap, hash_map::Entry}, - sync::Weak, + collections::HashMap, + fmt::{Debug, Formatter}, + hash::{Hash, Hasher}, + ops::Deref, }, }; #[repr(transparent)] -#[derive(Clone, Debug)] pub struct ProgramRuntimeEnvironment(Arc>>); -impl std::hash::Hash for ProgramRuntimeEnvironment { - fn hash(&self, state: &mut H) { - Arc::>>::as_ptr(&self.0).hash(state); - } -} -impl PartialEq for ProgramRuntimeEnvironment { - fn eq(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.0, &other.0) - } -} -impl Eq for ProgramRuntimeEnvironment {} -impl std::ops::Deref for ProgramRuntimeEnvironment { - type Target = Arc>>; - fn deref(&self) -> &Self::Target { - &self.0 - } -} impl ProgramRuntimeEnvironment { - pub fn from(inner: BuiltinProgram>) -> Self { - Self(Arc::new(inner)) + pub fn from(program: BuiltinProgram>) -> Self { + Self(Arc::new(program)) } - pub const fn from_ref<'a>( - inner: &'a Arc>>, + /// Converts a loader reference into its transparent runtime-environment wrapper. + /// + /// # Safety + /// + /// `ProgramRuntimeEnvironment` is `repr(transparent)` over the same `Arc` + /// type, so the reference layout is identical. + pub unsafe fn from_ref<'a>( + program: &'a Arc>>, ) -> &'a Self { - // Safety: This wrapper type is transparent and shares the same representation as the underlying type - unsafe { std::mem::transmute(inner) } + unsafe { &*(program as *const Arc<_> as *const Self) } } } -/// Paired execution and deployment environments. -/// -/// Registered functions within each program runtime environment (syscalls) -/// depend on per-epoch feature gate statuses. In most cases, the list of -/// registered functions in the two environments will be the same. However, -/// it's possible that the effective epoch of deployment could be in the -/// *next epoch*. -pub struct ProgramRuntimeEnvironments { - /// Environment compiled for the current epoch in which programs are - /// executing. - execution: ProgramRuntimeEnvironment, - /// Environment compiled for the epoch of the next slot at which a program - /// deployed in the current slot will execute. - deployment: ProgramRuntimeEnvironment, +impl Clone for ProgramRuntimeEnvironment { + fn clone(&self) -> Self { + Self(self.0.clone()) + } } -impl ProgramRuntimeEnvironments { - /// Create a new ProgramRuntimeEnvironments from an `execution` and - /// `deployment` environment. - pub fn new( - execution: ProgramRuntimeEnvironment, - deployment: ProgramRuntimeEnvironment, - ) -> Self { - Self { - execution, - deployment, - } +impl Debug for ProgramRuntimeEnvironment { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("ProgramRuntimeEnvironment").field(&Arc::as_ptr(&self.0)).finish() } +} - /// Get the program runtime environment for execution. - pub fn get_env_for_execution(&self) -> &ProgramRuntimeEnvironment { - &self.execution - } +impl Deref for ProgramRuntimeEnvironment { + type Target = Arc>>; - /// Get the program runtime environment for deployment. - pub fn get_env_for_deployment(&self) -> &ProgramRuntimeEnvironment { - &self.deployment + fn deref(&self) -> &Self::Target { + &self.0 } +} - #[cfg(feature = "dev-context-only-utils")] - pub fn mock() -> Self { - Self { - execution: get_mock_program_runtime_environment(), - deployment: get_mock_program_runtime_environment(), - } +impl Hash for ProgramRuntimeEnvironment { + fn hash(&self, state: &mut H) { + std::ptr::hash(Arc::as_ptr(&self.0), state); } } -#[cfg(feature = "dev-context-only-utils")] -pub fn get_mock_program_runtime_environment() -> ProgramRuntimeEnvironment { - static MOCK_ENVIRONMENT: std::sync::OnceLock = - std::sync::OnceLock::::new(); - MOCK_ENVIRONMENT - .get_or_init(|| ProgramRuntimeEnvironment::from(BuiltinProgram::new_mock())) - .clone() +impl PartialEq for ProgramRuntimeEnvironment { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } } +impl Eq for ProgramRuntimeEnvironment {} + pub const MAX_LOADED_ENTRY_COUNT: usize = 512; -/// Relationship between two fork IDs -#[derive(Copy, Clone, Debug, PartialEq)] -pub enum BlockRelation { - /// The slot is on the same fork and is an ancestor of the other slot - Ancestor, - /// The two slots are equal and are on the same fork - Equal, - /// The slot is on the same fork and is a descendant of the other slot - Descendant, - /// The slots are on two different forks and may have had a common ancestor at some point - Unrelated, - /// Either one or both of the slots are either older than the latest root, or are in future - Unknown, +/// Actual payload of [ProgramCacheEntry]. +#[derive(Default)] +pub enum ProgramCacheEntryType { + /// Program failed verification for the current runtime environment. + FailedVerification(ProgramRuntimeEnvironment), + /// Program is unavailable or intentionally closed. + #[default] + Closed, + /// Retained for API compatibility with older delayed-visibility flows. + DelayVisibility, + /// Program was verified but is not currently compiled. + Unloaded(ProgramRuntimeEnvironment), + /// Verified and compiled program. + Loaded(Executable>), + /// Builtin program shipped with the runtime. + Builtin(BuiltinProgram>), +} + +impl Debug for ProgramCacheEntryType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ProgramCacheEntryType::FailedVerification(_) => { + write!(f, "ProgramCacheEntryType::FailedVerification") + } + ProgramCacheEntryType::Closed => write!(f, "ProgramCacheEntryType::Closed"), + ProgramCacheEntryType::DelayVisibility => { + write!(f, "ProgramCacheEntryType::DelayVisibility") + } + ProgramCacheEntryType::Unloaded(_) => write!(f, "ProgramCacheEntryType::Unloaded"), + ProgramCacheEntryType::Loaded(_) => write!(f, "ProgramCacheEntryType::Loaded"), + ProgramCacheEntryType::Builtin(_) => write!(f, "ProgramCacheEntryType::Builtin"), + } + } } -/// Maps relationship between two slots. -pub trait ForkGraph { - /// Returns the BlockRelation of A to B - fn relationship(&self, a: Slot, b: Slot) -> BlockRelation; +impl ProgramCacheEntryType { + /// Returns the runtime environment when this entry keeps one. + pub fn get_environment(&self) -> Option<&ProgramRuntimeEnvironment> { + match self { + ProgramCacheEntryType::Loaded(program) => { + // SAFETY: `ProgramRuntimeEnvironment` is transparent over the loader Arc. + Some(unsafe { ProgramRuntimeEnvironment::from_ref(program.get_loader()) }) + } + ProgramCacheEntryType::FailedVerification(env) + | ProgramCacheEntryType::Unloaded(env) => Some(env), + _ => None, + } + } } -/// Globally manages the transition between environments at the epoch boundary +/// Single cache entry for a program address. #[derive(Debug, Default)] -pub struct EpochBoundaryPreparation { - /// The epoch of the upcoming_environment - pub upcoming_epoch: Epoch, - /// Anticipated replacement for `environments` at the next epoch - /// - /// This is `None` during most of an epoch, and only `Some` around the boundaries (at the end and beginning of an epoch). - /// More precisely, it starts with the cache preparation phase a few hundred slots before the epoch boundary, - /// and it ends with the first rerooting after the epoch boundary. - pub upcoming_environment: Option, - /// List of loaded programs which should be recompiled before the next epoch (but don't have to). - pub programs_to_recompile: Vec<(Pubkey, Arc)>, +pub struct ProgramCacheEntry { + pub program: ProgramCacheEntryType, } -impl EpochBoundaryPreparation { - pub fn new(epoch: Epoch) -> Self { - Self { - upcoming_epoch: epoch, - upcoming_environment: None, - programs_to_recompile: Vec::default(), - } +impl ProgramCacheEntry { + /// Creates a new user program. + pub fn new( + program_runtime_environment: ProgramRuntimeEnvironment, + elf_bytes: &[u8], + ) -> Result> { + Self::new_internal(program_runtime_environment, elf_bytes, false) } - /// Returns the upcoming environments depending on the given epoch - pub fn get_upcoming_environment_for_epoch( - &self, - epoch: Epoch, - ) -> Option { - if epoch == self.upcoming_epoch { - return self.upcoming_environment.clone(); + /// Reloads a previously verified user program without re-running the verifier. + /// + /// # Safety + /// + /// Callers must ensure `elf_bytes` were already verified for the provided + /// runtime environment. + pub unsafe fn reload( + program_runtime_environment: ProgramRuntimeEnvironment, + elf_bytes: &[u8], + ) -> Result> { + Self::new_internal(program_runtime_environment, elf_bytes, true) + } + + fn new_internal( + program_runtime_environment: ProgramRuntimeEnvironment, + elf_bytes: &[u8], + reloading: bool, + ) -> Result> { + // Some architectures build without JIT support. + #[allow(unused_mut)] + let mut executable = + Executable::load(elf_bytes, Arc::clone(&*program_runtime_environment))?; + if !reloading { + executable.verify::()?; } - None + + #[cfg(all(not(target_os = "windows"), target_arch = "x86_64"))] + executable.jit_compile()?; + + Ok(Self { + program: ProgramCacheEntryType::Loaded(executable), + }) } - /// Before rerooting the blockstore this concludes the epoch boundary preparation - pub fn reroot(&mut self, epoch: Epoch) -> Option { - if epoch == self.upcoming_epoch - && let Some(upcoming_environment) = self.upcoming_environment.take() - { - self.programs_to_recompile.clear(); - return Some(upcoming_environment); + /// Creates a new built-in program. + pub fn new_builtin(builtin_function: BuiltinFunctionWithContext) -> Self { + let mut program = BuiltinProgram::new_builtin(); + program.register_function("entrypoint", builtin_function).unwrap(); + Self { + program: ProgramCacheEntryType::Builtin(program), } + } +} + +/// Shared runtime environments keyed by feature configuration. +#[derive(Clone, Debug)] +pub struct ProgramRuntimeEnvironments { + execution: ProgramRuntimeEnvironment, +} + +impl ProgramRuntimeEnvironments { + pub fn new(execution: ProgramRuntimeEnvironment) -> Self { + Self { execution } + } - None + pub fn get_env_for_execution(&self) -> &ProgramRuntimeEnvironment { + &self.execution } } -#[derive(Debug)] -pub(crate) enum IndexImplementation { - /// Fork-graph aware index implementation - V1 { - /// A two level index: - /// - /// - the first level is for the address at which programs are deployed - /// - the second level for the slot (and thus also fork), sorted by slot number. - entries: HashMap>>, - /// The entries that are getting loaded and have not yet finished loading. - /// - /// The key is the program address, the value is a tuple of the slot in which the program is - /// being loaded and the thread ID doing the load. - /// - /// It is possible that multiple TX batches from different slots need different versions of a - /// program. The deployment slot of a program is only known after load tho, - /// so all loads for a given program key are serialized. - loading_entries: Mutex>, - }, +impl Default for ProgramRuntimeEnvironments { + fn default() -> Self { + let empty_loader = + ProgramRuntimeEnvironment::from(BuiltinProgram::new_loader(Config::default())); + Self::new(empty_loader.clone()) + } } -/// This structure is the global cache of loaded, verified and compiled programs. -/// -/// It ... -/// - is validator global and fork graph aware, so it can optimize the commonalities across banks. -/// - handles the visibility rules of un/re/deployments. -/// - stores the usage statistics and verification status of each program. -/// - is elastic and uses a probabilistic eviction strategy based on the usage statistics. -/// - also keeps the compiled executables around, but only for the most used programs. -/// - supports various kinds of tombstones to avoid loading programs which can not be loaded. -/// - cleans up entries on orphan branches when the block store is rerooted. -/// - supports the cache preparation phase before feature activations which can change cached programs. -/// - manages the environments of the programs and upcoming environments for the next epoch. -/// - allows for cooperative loading of TX batches which hit the same missing programs simultaneously. -/// - enforces that all programs used in a batch are eagerly loaded ahead of execution. -/// - is not persisted to disk or a snapshot, so it needs to cold start and warm up first. -pub struct ProgramCache { - /// Index of the cached entries and cooperative loading tasks - pub(crate) index: IndexImplementation, - /// The slot of the last rerooting - pub latest_root_slot: Slot, - /// Statistics counters - pub stats: ProgramCacheStats, - /// Reference to the block store - pub fork_graph: Option>>, - /// Coordinates TX batches waiting for others to complete their task during cooperative loading - pub loading_task_waiter: Arc, +/// Global program cache shared across transaction batches. +#[derive(Default)] +pub struct ProgramCache { + index: scc::HashMap>, } -impl std::fmt::Debug for ProgramCache { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ProgramCache") - .field("root slot", &self.latest_root_slot) - .field("stats", &self.stats) - .field("index", &self.index) - .finish() +impl Debug for ProgramCache { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProgramCache").field("index_len", &self.index.len()).finish() } } -/// Local view into [ProgramCache] which was extracted for a specific TX batch. -/// -/// This isolation enables the global [ProgramCache] to continue to evolve (e.g. evictions), -/// while the TX batch is guaranteed it will continue to find all the programs it requires. -/// For program management instructions this also buffers them before they are merged back into the global [ProgramCache]. +/// Local view into [ProgramCache] used by a transaction batch. #[derive(Clone, Debug, Default)] pub struct ProgramCacheForTxBatch { - /// Pubkey is the address of a program. - /// ProgramCacheEntry is the corresponding program entry valid for the slot in which a transaction is being executed. entries: HashMap>, - /// Program entries modified during the transaction batch. modified_entries: HashMap>, slot: Slot, pub hit_max_limit: bool, @@ -272,53 +239,20 @@ impl ProgramCacheForTxBatch { } } - /// Refill the cache with a single entry. It's typically called during transaction loading, and - /// transaction processing (for program management instructions). - /// It replaces the existing entry (if any) with the provided entry. The return value contains - /// `true` if an entry existed. - /// The function also returns the newly inserted value. - pub fn replenish( - &mut self, - key: Pubkey, - entry: Arc, - ) -> (bool, Arc) { - (self.entries.insert(key, entry.clone()).is_some(), entry) + pub fn replenish(&mut self, key: Pubkey, entry: Arc) { + self.entries.insert(key, entry); } - /// Store an entry in `modified_entries` for a program modified during the - /// transaction batch. pub fn store_modified_entry(&mut self, key: Pubkey, entry: Arc) { self.modified_entries.insert(key, entry); } - /// Drain the program cache's modified entries, returning the owned - /// collection. pub fn drain_modified_entries(&mut self) -> HashMap> { std::mem::take(&mut self.modified_entries) } pub fn find(&self, key: &Pubkey) -> Option> { - // First lookup the cache of the programs modified by the current - // transaction. If not found, lookup the cache of the cache of the - // programs that are loaded for the transaction batch. - self.modified_entries - .get(key) - .or_else(|| self.entries.get(key)) - .map(|entry| { - if entry.is_implicit_delay_visibility_tombstone(self.slot) { - // Found a program entry on the current fork, but it's not effective - // yet. It indicates that the program has delayed visibility. Return - // the tombstone to reflect that. - Arc::new(ProgramCacheEntry::new_tombstone_with_stats( - entry.deployment_slot, - entry.account_owner, - ProgramCacheEntryType::DelayVisibility, - Arc::clone(&entry.stats), - )) - } else { - entry.clone() - } - }) + self.modified_entries.get(key).or_else(|| self.entries.get(key)).cloned() } pub fn slot(&self) -> Slot { @@ -330,10 +264,10 @@ impl ProgramCacheForTxBatch { } pub fn merge(&mut self, modified_entries: &HashMap>) { - modified_entries.iter().for_each(|(key, entry)| { + for (key, entry) in modified_entries { self.merged_modified = true; self.replenish(*key, entry.clone()); - }) + } } pub fn is_empty(&self) -> bool { @@ -341,2140 +275,64 @@ impl ProgramCacheForTxBatch { } } -pub enum ProgramCacheMatchCriteria { - DeployedOnOrAfterSlot(Slot), - Tombstone, - NoCriteria, -} - -impl ProgramCache { - pub fn new(root_slot: Slot) -> Self { - Self { - index: IndexImplementation::V1 { - entries: HashMap::new(), - loading_entries: Mutex::new(HashMap::new()), - }, - latest_root_slot: root_slot, - stats: ProgramCacheStats::default(), - fork_graph: None, - loading_task_waiter: Arc::new(LoadingTaskWaiter::default()), - } - } - - pub fn set_fork_graph(&mut self, fork_graph: Weak>) { - self.fork_graph = Some(fork_graph); - } - - /// Insert a single entry. It's typically called during transaction loading, - /// when the cache doesn't contain the entry corresponding to program `key`. - pub fn assign_program( - &mut self, - program_runtime_environment: &ProgramRuntimeEnvironment, - key: Pubkey, - _last_modification_slot: Slot, - entry: Arc, - ) -> bool { - debug_assert!(!matches!( - &entry.program, - ProgramCacheEntryType::DelayVisibility - )); - // This function always returns `true` during normal operation. - // Only during the cache preparation phase this can return `false` - // for entries with `upcoming_environment`. - fn is_current_env( - program_runtime_environment: &ProgramRuntimeEnvironment, - env_opt: Option<&ProgramRuntimeEnvironment>, - ) -> bool { - env_opt - .map(|env| env == program_runtime_environment) - .unwrap_or(true) - } - match &mut self.index { - IndexImplementation::V1 { entries, .. } => { - let slot_versions = &mut entries.entry(key).or_default(); - let insertion_point = slot_versions.binary_search_by(|at| { - at.effective_slot - .cmp(&entry.effective_slot) - .then(at.deployment_slot.cmp(&entry.deployment_slot)) - .then( - // This `.then()` has no effect during normal operation. - // Only during the cache preparation phase this does allow entries - // which only differ in their environment to be interleaved in `slot_versions`. - is_current_env( - program_runtime_environment, - at.program.get_environment(), - ) - .cmp(&is_current_env( - program_runtime_environment, - entry.program.get_environment(), - )), - ) - }); - match insertion_point { - Ok(index) => { - let existing = slot_versions.get_mut(index).unwrap(); - match (&existing.program, &entry.program) { - ( - ProgramCacheEntryType::Builtin(_), - ProgramCacheEntryType::Builtin(_), - ) - | ( - ProgramCacheEntryType::Unloaded(_), - ProgramCacheEntryType::Loaded(_), - ) => {} - (ProgramCacheEntryType::Closed, ProgramCacheEntryType::Closed) - if existing.account_owner != entry.account_owner => {} - _ => { - // Something is wrong, I can feel it ... - error!( - "ProgramCache::assign_program() failed key={key:?} \ - existing={slot_versions:?} entry={entry:?}" - ); - debug_assert!(false, "Unexpected replacement of an entry"); - self.stats.replacements.fetch_add(1, Ordering::Relaxed); - return true; - } - } - entry.stats.merge_from(&existing.stats); - *existing = Arc::clone(&entry); - self.stats.reloads.fetch_add(1, Ordering::Relaxed); - } - Err(index) => { - self.stats.insertions.fetch_add(1, Ordering::Relaxed); - slot_versions.insert(index, Arc::clone(&entry)); - } - } - // Remove existing entries in the same deployment slot unless they are for a different - // environment. - // This overwrites the current status of a program in program management instructions. - slot_versions.retain(|existing| { - existing.deployment_slot != entry.deployment_slot - || existing - .program - .get_environment() - .zip(entry.program.get_environment()) - .map(|(a, b)| a != b) - .unwrap_or(false) - || existing == &entry - }); - } - } - false - } - - pub fn prune_by_deployment_slot(&mut self, slot: Slot) { - match &mut self.index { - IndexImplementation::V1 { entries, .. } => { - for second_level in entries.values_mut() { - second_level.retain(|entry| entry.deployment_slot != slot); - } - self.remove_programs_with_no_entries(); - } - } - } - - /// Before rerooting the blockstore this removes all superfluous entries - pub fn prune( - &mut self, - new_root_slot: Slot, - upcoming_environment: Option, - fork_graph: &FG, - ) { - match &mut self.index { - IndexImplementation::V1 { entries, .. } => { - for second_level in entries.values_mut() { - // Remove entries un/re/deployed on orphan forks - let mut first_ancestor_found = false; - let mut first_ancestor_env = None; - *second_level = second_level - .iter() - .rev() - .filter(|entry| { - let relation = - fork_graph.relationship(entry.deployment_slot, new_root_slot); - if entry.deployment_slot >= new_root_slot { - matches!(relation, BlockRelation::Equal | BlockRelation::Descendant) - } else if matches!(relation, BlockRelation::Ancestor) - || entry.deployment_slot <= self.latest_root_slot - { - if !first_ancestor_found { - first_ancestor_found = true; - first_ancestor_env = entry.program.get_environment(); - return true; - } - // Do not prune the entry if the runtime environment of the entry is - // different than the entry that was previously found (stored in - // first_ancestor_env). Different environment indicates that this entry - // might belong to an older epoch that had a different environment (e.g. - // different feature set). Once the root moves to the new/current epoch, - // the entry will get pruned. But, until then the entry might still be - // getting used by an older slot. - if let Some(entry_env) = entry.program.get_environment() - && let Some(env) = first_ancestor_env - && entry_env != env - { - return true; - } - self.stats.prunes_orphan.fetch_add(1, Ordering::Relaxed); - false - } else { - self.stats.prunes_orphan.fetch_add(1, Ordering::Relaxed); - false - } - }) - .filter(|entry| { - // Remove outdated environment of previous feature set - if let Some(upcoming_environment) = upcoming_environment.as_ref() - && !Self::matches_environment(entry, upcoming_environment) - { - self.stats - .prunes_environment - .fetch_add(1, Ordering::Relaxed); - return false; - } - true - }) - .cloned() - .collect(); - second_level.reverse(); - } - } - } - self.remove_programs_with_no_entries(); - debug_assert!(self.latest_root_slot <= new_root_slot); - self.latest_root_slot = new_root_slot; +impl ProgramCache { + pub fn assign_program(&self, key: Pubkey, entry: Arc) { + self.index.upsert_sync(key, entry); } - fn matches_environment( - entry: &Arc, - program_runtime_environment: &ProgramRuntimeEnvironment, - ) -> bool { - let Some(environment) = entry.program.get_environment() else { - return true; - }; - environment == program_runtime_environment + pub fn get(&self, key: &Pubkey) -> Option> { + self.index.get_sync(key).map(|e| e.get().clone()) } - fn matches_criteria( - program: &Arc, - criteria: &ProgramCacheMatchCriteria, - ) -> bool { - match criteria { - ProgramCacheMatchCriteria::DeployedOnOrAfterSlot(slot) => { - program.deployment_slot >= *slot - } - ProgramCacheMatchCriteria::Tombstone => program.is_tombstone(), - ProgramCacheMatchCriteria::NoCriteria => true, + pub fn merge(&self, modified_entries: &HashMap>) { + for (key, entry) in modified_entries { + self.assign_program(*key, entry.clone()); } } - - /// Extracts a subset of the programs relevant to a transaction batch - /// and returns which program accounts the accounts DB needs to load. - pub fn extract( - &self, - search_for: &mut Vec<(Pubkey, ProgramCacheMatchCriteria, Slot)>, - loaded_programs_for_tx_batch: &mut ProgramCacheForTxBatch, - program_runtime_environment_for_execution: &ProgramRuntimeEnvironment, - increment_usage_counter: bool, - count_hits_and_misses: bool, - ) -> Option { - debug_assert!(self.fork_graph.is_some()); - let fork_graph = self.fork_graph.as_ref().unwrap().upgrade().unwrap(); - let locked_fork_graph = fork_graph.read().unwrap(); - let mut cooperative_loading_task = None; - match &self.index { - IndexImplementation::V1 { - entries, - loading_entries, - } => { - search_for.retain(|(key, match_criteria, _slot)| { - if let Some(second_level) = entries.get(key) { - let mut filter_by_deployment_slot = None; - for entry in second_level.iter().rev() { - let required_deployment_slot = - filter_by_deployment_slot.unwrap_or(entry.deployment_slot); - if required_deployment_slot != entry.deployment_slot { - continue; - } - let entry_in_same_branch = entry.deployment_slot - <= self.latest_root_slot - || matches!( - locked_fork_graph.relationship( - entry.deployment_slot, - loaded_programs_for_tx_batch.slot - ), - BlockRelation::Equal | BlockRelation::Ancestor - ); - if entry_in_same_branch { - let entry_is_effective = - loaded_programs_for_tx_batch.slot >= entry.effective_slot; - let entry_to_return = if entry_is_effective { - if !Self::matches_environment( - entry, - program_runtime_environment_for_execution, - ) { - // We found an entry that would work, had its environment matched - // the one we're planning to use for this slot. - // - // At this point we know that whatever the "current version" of - // program is, it must have had a deployment slot equal to the - // program we're looking at in this iteration. We just have to find - // one with the correct environment and can skip entries for any - // other deployment slot while searching further. - filter_by_deployment_slot = filter_by_deployment_slot - .or(Some(entry.deployment_slot)); - continue; - } - if !Self::matches_criteria(entry, match_criteria) { - break; - } - if let ProgramCacheEntryType::Unloaded(_environment) = - &entry.program - { - break; - } - entry.clone() - } else if entry.is_implicit_delay_visibility_tombstone( - loaded_programs_for_tx_batch.slot, - ) { - // Found a program entry on the current fork, but it's not effective - // yet. It indicates that the program has delayed visibility. Return - // the tombstone to reflect that. - Arc::new(ProgramCacheEntry::new_tombstone_with_stats( - entry.deployment_slot, - entry.account_owner, - ProgramCacheEntryType::DelayVisibility, - Arc::clone(&entry.stats), - )) - } else { - continue; - }; - entry_to_return - .update_access_slot(loaded_programs_for_tx_batch.slot); - if increment_usage_counter { - entry_to_return.stats.uses.fetch_add(1, Ordering::Relaxed); - } - loaded_programs_for_tx_batch - .entries - .insert(*key, entry_to_return); - return false; - } - } - } - if cooperative_loading_task.is_none() { - let mut loading_entries = loading_entries.lock().unwrap(); - let entry = loading_entries.entry(*key); - if let Entry::Vacant(entry) = entry { - entry.insert(( - loaded_programs_for_tx_batch.slot, - thread::current().id(), - )); - cooperative_loading_task = Some(*key); - } - } - true - }); - } - } - drop(locked_fork_graph); - if count_hits_and_misses { - self.stats - .misses - .fetch_add(search_for.len() as u64, Ordering::Relaxed); - self.stats.hits.fetch_add( - loaded_programs_for_tx_batch.entries.len() as u64, - Ordering::Relaxed, - ); - } - cooperative_loading_task - } - - /// Called by Bank::replenish_program_cache() for each program that is done loading. - pub fn finish_cooperative_loading_task( - &mut self, - program_runtime_environment: &ProgramRuntimeEnvironment, - current_slot: Slot, - key: Pubkey, - last_modification_slot: Slot, - loaded_program: Arc, - ) -> bool { - match &mut self.index { - IndexImplementation::V1 { - loading_entries, .. - } => { - let loading_thread = loading_entries.get_mut().unwrap().remove(&key); - debug_assert_eq!(loading_thread, Some((current_slot, thread::current().id()))); - // Check that it will be visible to our own fork once inserted - if loaded_program.deployment_slot > self.latest_root_slot - && !matches!( - self.fork_graph - .as_ref() - .unwrap() - .upgrade() - .unwrap() - .read() - .unwrap() - .relationship(loaded_program.deployment_slot, current_slot), - BlockRelation::Equal | BlockRelation::Ancestor - ) - { - self.stats.lost_insertions.fetch_add(1, Ordering::Relaxed); - } - let was_occupied = self.assign_program( - program_runtime_environment, - key, - last_modification_slot, - loaded_program, - ); - self.loading_task_waiter.notify(); - was_occupied - } - } - } - - pub fn merge( - &mut self, - program_runtime_environment: &ProgramRuntimeEnvironment, - current_slot: Slot, - modified_entries: &HashMap>, - ) { - modified_entries.iter().for_each(|(key, entry)| { - self.assign_program( - program_runtime_environment, - *key, - current_slot, - entry.clone(), - ); - }) - } - - /// Returns the list of entries which are verified and compiled. - pub fn get_flattened_entries(&self) -> Vec<(Pubkey, Slot, Arc)> { - match &self.index { - IndexImplementation::V1 { entries, .. } => entries - .iter() - .flat_map(|(id, second_level)| { - second_level - .iter() - .filter_map(move |program| match program.program { - ProgramCacheEntryType::Loaded(_) => Some((*id, 0, program.clone())), - _ => None, - }) - }) - .collect(), - } - } - - /// Returns the list of all entries in the cache. - #[cfg(feature = "dev-context-only-utils")] - pub fn get_flattened_entries_for_tests(&self) -> Vec<(Pubkey, Arc)> { - match &self.index { - IndexImplementation::V1 { entries, .. } => entries - .iter() - .flat_map(|(id, second_level)| { - second_level.iter().map(|program| (*id, program.clone())) - }) - .collect(), - } - } - - /// Returns the slot versions for the given program id. - pub fn get_slot_versions_for_tests(&self, key: &Pubkey) -> &[Arc] { - match &self.index { - IndexImplementation::V1 { entries, .. } => entries - .get(key) - .map(|second_level| second_level.as_ref()) - .unwrap_or(&[]), - } - } - - /// Unloads programs which were used infrequently - pub fn sort_and_unload(&mut self, shrink_to: PercentageInteger) { - let mut sorted_candidates = self.get_flattened_entries(); - sorted_candidates.sort_by_cached_key(|(_id, _last_modification_slot, program)| { - program.stats.uses.load(Ordering::Relaxed) - }); - let num_to_unload = sorted_candidates - .len() - .saturating_sub(shrink_to.apply_to(MAX_LOADED_ENTRY_COUNT)); - for (program, last_modification_slot, entry) in sorted_candidates.iter().take(num_to_unload) - { - self.unload_program_entry(*program, *last_modification_slot, entry); - } - } - - /// Evicts programs using random selection, choosing the worst scoring program out of the - /// entries sampled. - /// - /// The eviction is performed enough number of times to reduce the cache usage to the given - /// percentage. - pub fn evict_using_random_selection(&mut self, shrink_to: PercentageInteger, now: Slot) { - let mut candidates = self.get_flattened_entries(); - let mut rng = rng(); - self.stats - .water_level - .store(candidates.len() as u64, Ordering::Relaxed); - let num_to_unload = candidates - .len() - .saturating_sub(shrink_to.apply_to(MAX_LOADED_ENTRY_COUNT)); - let mut sample_entry = |candidates: &Vec<(Pubkey, u64, Arc)>| { - // gen_range is deprecated in favor of random_range in rand>=0.9, but we also get - // rnd() from shuttle, which doesn't yet support rand 0.9 APIs - #[cfg(feature = "shuttle-test")] - let index = rng.gen_range(0..candidates.len()); - #[cfg(not(feature = "shuttle-test"))] - let index = rng.random_range(0..candidates.len()); - let usage_counter = candidates - .get(index) - .expect("Failed to get cached entry") - .2 - .retention_score(); - (index, usage_counter) - }; - - // Random sampling with just 2 choices can frequently lead to a situation where both - // entries chosen have relatively high retention scores, having us to pick one out of two - // poor options. We can tell what a relatively high retention score is, so we can make a - // few additional samples until we hit some other entry that isn't as highly scoring. - // - // Note that the "high enough" compilation time and use count numbers used here are - // relatively arbitrary. - const MAX_ADDITIONAL_SAMPLES: usize = 3; - let avoid_evicting_above_score = retention_score(now, 500 * EMA_SCALE, 500); - for _ in 0..num_to_unload { - let (mut index, mut score) = sample_entry(&candidates); - for _ in 0..MAX_ADDITIONAL_SAMPLES { - let (sample_index, sample_score) = sample_entry(&candidates); - if score > sample_score { - index = sample_index; - score = sample_score; - } - if score < avoid_evicting_above_score { - break; - } - } - let (id, last_modification_slot, entry) = candidates.swap_remove(index); - self.unload_program_entry(id, last_modification_slot, &entry); - } - } - - /// Removes all the entries at the given keys, if they exist - pub fn remove_programs(&mut self, keys: impl Iterator) { - match &mut self.index { - IndexImplementation::V1 { entries, .. } => { - for k in keys { - entries.remove(&k); - } - } - } - } - - /// This function removes the given entry for the given program from the cache. - /// The function expects that the program and entry exists in the cache. Otherwise it'll panic. - fn unload_program_entry( - &mut self, - id: Pubkey, - _last_modification_slot: Slot, - remove_entry: &Arc, - ) { - match &mut self.index { - IndexImplementation::V1 { entries, .. } => { - let second_level = entries.get_mut(&id).expect("Cache lookup failed"); - let candidate = second_level - .iter_mut() - .find(|entry| entry == &remove_entry) - .expect("Program entry not found"); - - // Certain entry types cannot be unloaded, such as tombstones, or already unloaded entries. - // For such entries, `to_unloaded()` will return None. - // These entry types do not occupy much memory. - if let Some(unloaded) = candidate.to_unloaded() { - if candidate.stats.uses.load(Ordering::Relaxed) == 1 { - self.stats.one_hit_wonders.fetch_add(1, Ordering::Relaxed); - } - self.stats - .evictions - .entry(id) - .and_modify(|c| *c = c.saturating_add(1)) - .or_insert(1); - *candidate = Arc::new(unloaded); - } - } - } - } - - fn remove_programs_with_no_entries(&mut self) { - match &mut self.index { - IndexImplementation::V1 { entries, .. } => { - let num_programs_before_removal = entries.len(); - entries.retain(|_key, second_level| !second_level.is_empty()); - if entries.len() < num_programs_before_removal { - self.stats.empty_entries.fetch_add( - num_programs_before_removal.saturating_sub(entries.len()) as u64, - Ordering::Relaxed, - ); - } - } - } - } -} - -#[cfg(feature = "frozen-abi")] -impl solana_frozen_abi::abi_example::AbiExample for ProgramCacheEntry { - fn example() -> Self { - // ProgramCacheEntry isn't serializable by definition. - Self::default() - } -} - -#[cfg(feature = "frozen-abi")] -impl solana_frozen_abi::abi_example::AbiExample for ProgramCache { - fn example() -> Self { - // ProgramCache isn't serializable by definition. - Self::new(Slot::default()) - } } #[cfg(test)] -pub(crate) mod tests { +mod tests { use { - crate::{ - loaded_programs::{ - BlockRelation, ForkGraph, ProgramCache, ProgramCacheForTxBatch, - ProgramCacheMatchCriteria, ProgramRuntimeEnvironment, - get_mock_program_runtime_environment, - }, - program_cache_entry::{ - DELAY_VISIBILITY_SLOT_OFFSET, ProgramCacheEntry, ProgramCacheEntryOwner, - ProgramCacheEntryType, - }, - program_metrics::ProgramStatistics, + super::{ + ProgramCache, ProgramCacheEntry, ProgramCacheEntryType, ProgramRuntimeEnvironment, }, - assert_matches::assert_matches, - percentage::Percentage, - solana_clock::Slot, solana_pubkey::Pubkey, solana_sbpf::{elf::Executable, program::BuiltinProgram}, - std::{ - fs::File, - io::Read, - ops::ControlFlow, - sync::{ - Arc, RwLock, - atomic::{AtomicU64, Ordering}, - }, - }, - test_case::{test_case, test_matrix}, + solana_svm_type_overrides::sync::Arc, }; - fn new_test_entry(deployment_slot: Slot, effective_slot: Slot) -> Arc { - new_test_entry_with_usage( - deployment_slot, - effective_slot, - ProgramStatistics::default(), - ) - } - - fn new_loaded_entry(env: ProgramRuntimeEnvironment) -> ProgramCacheEntryType { - let mut elf = Vec::new(); - File::open("../programs/bpf_loader/test_elfs/out/noop_aligned.so") - .unwrap() - .read_to_end(&mut elf) - .unwrap(); - let executable = Executable::load(&elf, Arc::clone(&*env)).unwrap(); - ProgramCacheEntryType::Loaded(executable) - } - - pub(crate) fn new_test_entry_with_usage( - deployment_slot: Slot, - effective_slot: Slot, - stats: ProgramStatistics, - ) -> Arc { - Arc::new(ProgramCacheEntry { - program: new_loaded_entry(get_mock_program_runtime_environment()), - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot, - effective_slot, - stats: Arc::new(stats), - latest_access_slot: AtomicU64::new(deployment_slot), - }) - } - - fn new_test_builtin_entry( - deployment_slot: Slot, - effective_slot: Slot, - ) -> Arc { + static MOCK_ENVIRONMENT: std::sync::OnceLock = + std::sync::OnceLock::new(); + + fn mock_env() -> ProgramRuntimeEnvironment { + MOCK_ENVIRONMENT + .get_or_init(|| ProgramRuntimeEnvironment::from(BuiltinProgram::new_mock())) + .clone() + } + + fn test_entry() -> Arc { + let elf = std::fs::read(concat!( + env!("CARGO_MANIFEST_DIR"), + "/fixtures/noop_aligned.so" + )) + .unwrap(); + let environment = mock_env(); + let executable = Executable::load(&elf, Arc::clone(&*environment)).unwrap(); Arc::new(ProgramCacheEntry { - program: ProgramCacheEntryType::Builtin(BuiltinProgram::new_mock()), - account_owner: ProgramCacheEntryOwner::NativeLoader, - account_size: 0, - deployment_slot, - effective_slot, - stats: Arc::default(), - latest_access_slot: AtomicU64::default(), + program: ProgramCacheEntryType::Loaded(executable), }) } - fn set_tombstone( - cache: &mut ProgramCache, - key: Pubkey, - current_slot: Slot, - reason: ProgramCacheEntryType, - ) -> Arc { - let env = get_mock_program_runtime_environment(); - let program = Arc::new(ProgramCacheEntry::new_tombstone( - current_slot, - ProgramCacheEntryOwner::LoaderV2, - reason, - )); - cache.assign_program(&env, key, current_slot, program.clone()); - program - } - - fn insert_unloaded_entry( - cache: &mut ProgramCache, - key: Pubkey, - current_slot: Slot, - ) -> Arc { - let env = get_mock_program_runtime_environment(); - let loaded = new_test_entry_with_usage( - current_slot, - current_slot.saturating_add(1), - ProgramStatistics::default(), - ); - let unloaded = Arc::new(loaded.to_unloaded().expect("Failed to unload the program")); - cache.assign_program(&env, key, current_slot, unloaded.clone()); - unloaded - } - - fn num_matching_entries(cache: &ProgramCache, predicate: P) -> usize - where - P: Fn(&ProgramCacheEntryType) -> bool, - FG: ForkGraph, - { - cache - .get_flattened_entries_for_tests() - .iter() - .filter(|(_key, program)| predicate(&program.program)) - .count() - } - - fn program_deploy_test_helper( - cache: &mut ProgramCache, - program: Pubkey, - deployment_slots: Vec, - usage_counters: Vec, - programs: &mut Vec<(Pubkey, Slot, u64)>, - ) { - let env = get_mock_program_runtime_environment(); - // Add multiple entries for program - deployment_slots - .iter() - .enumerate() - .for_each(|(i, deployment_slot)| { - let usage_counter = *usage_counters.get(i).unwrap_or(&0); - let stats = ProgramStatistics { - uses: usage_counter.into(), - ..Default::default() - }; - cache.assign_program( - &env, - program, - *deployment_slot, - new_test_entry_with_usage( - *deployment_slot, - (*deployment_slot).saturating_add(2), - stats, - ), - ); - programs.push((program, *deployment_slot, usage_counter)); - }); - - // Add tombstones entries for program - let env = ProgramRuntimeEnvironment::from(BuiltinProgram::new_mock()); - for slot in 21..31 { - set_tombstone( - cache, - program, - slot, - ProgramCacheEntryType::FailedVerification(env.clone()), - ); - } - - // Add unloaded entries for program - for slot in 31..41 { - insert_unloaded_entry(cache, program, slot); - } - } - - #[test] - fn test_random_eviction() { - let mut programs = vec![]; - let mut cache = ProgramCache::::new(0); - - // This test adds different kind of entries to the cache. - // Tombstones and unloaded entries are expected to not be evicted. - // It also adds multiple entries for three programs as it tries to create a typical cache instance. - - // Program 1 - program_deploy_test_helper( - &mut cache, - Pubkey::new_unique(), - vec![0, 10, 20], - vec![4, 5, 25], - &mut programs, - ); - - // Program 2 - program_deploy_test_helper( - &mut cache, - Pubkey::new_unique(), - vec![5, 11], - vec![0, 2], - &mut programs, - ); - - // Program 3 - program_deploy_test_helper( - &mut cache, - Pubkey::new_unique(), - vec![0, 5, 15], - vec![100, 3, 20], - &mut programs, - ); - - // 1 for each deployment slot - let num_loaded_expected = 8; - // 10 for each program - let num_unloaded_expected = 30; - // 10 for each program - let num_tombstones_expected = 30; - - // Count the number of loaded, unloaded and tombstone entries. - programs.sort_by_key(|(_id, _slot, usage_count)| *usage_count); - let num_loaded = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::Loaded(_)) - }); - let num_unloaded = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::Unloaded(_)) - }); - let num_tombstones = num_matching_entries(&cache, |program_type| { - matches!( - program_type, - ProgramCacheEntryType::DelayVisibility - | ProgramCacheEntryType::FailedVerification(_) - | ProgramCacheEntryType::Closed - ) - }); - - // Test that the cache is constructed with the expected number of entries. - assert_eq!(num_loaded, num_loaded_expected); - assert_eq!(num_unloaded, num_unloaded_expected); - assert_eq!(num_tombstones, num_tombstones_expected); - - // Evict entries from the cache - let eviction_pct = 1; - - let num_loaded_expected = - Percentage::from(eviction_pct).apply_to(crate::loaded_programs::MAX_LOADED_ENTRY_COUNT); - let num_unloaded_expected = num_unloaded_expected + num_loaded - num_loaded_expected; - cache.evict_using_random_selection(Percentage::from(eviction_pct), 21); - - // Count the number of loaded, unloaded and tombstone entries. - let num_loaded = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::Loaded(_)) - }); - let num_unloaded = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::Unloaded(_)) - }); - let num_tombstones = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::FailedVerification(_)) - }); - - // However many entries are left after the shrink - assert_eq!(num_loaded, num_loaded_expected); - // The original unloaded entries + the evicted loaded entries - assert_eq!(num_unloaded, num_unloaded_expected); - // The original tombstones are not evicted - assert_eq!(num_tombstones, num_tombstones_expected); - } - - #[test] - fn test_eviction() { - let mut programs = vec![]; - let mut cache = ProgramCache::::new(0); - - // Program 1 - program_deploy_test_helper( - &mut cache, - Pubkey::new_unique(), - vec![0, 10, 20], - vec![4, 5, 25], - &mut programs, - ); - - // Program 2 - program_deploy_test_helper( - &mut cache, - Pubkey::new_unique(), - vec![5, 11], - vec![0, 2], - &mut programs, - ); - - // Program 3 - program_deploy_test_helper( - &mut cache, - Pubkey::new_unique(), - vec![0, 5, 15], - vec![100, 3, 20], - &mut programs, - ); - - // 1 for each deployment slot - let num_loaded_expected = 8; - // 10 for each program - let num_unloaded_expected = 30; - // 10 for each program - let num_tombstones_expected = 30; - - // Count the number of loaded, unloaded and tombstone entries. - programs.sort_by_key(|(_id, _slot, usage_count)| *usage_count); - let num_loaded = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::Loaded(_)) - }); - let num_unloaded = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::Unloaded(_)) - }); - let num_tombstones = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::FailedVerification(_)) - }); - - // Test that the cache is constructed with the expected number of entries. - assert_eq!(num_loaded, num_loaded_expected); - assert_eq!(num_unloaded, num_unloaded_expected); - assert_eq!(num_tombstones, num_tombstones_expected); - - // Evict entries from the cache - let eviction_pct = 1; - - let num_loaded_expected = - Percentage::from(eviction_pct).apply_to(crate::loaded_programs::MAX_LOADED_ENTRY_COUNT); - let num_unloaded_expected = num_unloaded_expected + num_loaded - num_loaded_expected; - - cache.sort_and_unload(Percentage::from(eviction_pct)); - - // Check that every program is still in the cache. - let entries = cache.get_flattened_entries_for_tests(); - programs.iter().for_each(|entry| { - assert!(entries.iter().any(|(key, _entry)| key == &entry.0)); - }); - - let unloaded = entries - .iter() - .filter_map(|(key, program)| { - matches!(program.program, ProgramCacheEntryType::Unloaded(_)) - .then_some((*key, program.stats.uses.load(Ordering::Relaxed))) - }) - .collect::>(); - - for index in 0..3 { - let expected = programs.get(index).expect("Missing program"); - assert!(unloaded.contains(&(expected.0, expected.2))); - } - - // Count the number of loaded, unloaded and tombstone entries. - let num_loaded = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::Loaded(_)) - }); - let num_unloaded = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::Unloaded(_)) - }); - let num_tombstones = num_matching_entries(&cache, |program_type| { - matches!( - program_type, - ProgramCacheEntryType::DelayVisibility - | ProgramCacheEntryType::FailedVerification(_) - | ProgramCacheEntryType::Closed - ) - }); - - // However many entries are left after the shrink - assert_eq!(num_loaded, num_loaded_expected); - // The original unloaded entries + the evicted loaded entries - assert_eq!(num_unloaded, num_unloaded_expected); - // The original tombstones are not evicted - assert_eq!(num_tombstones, num_tombstones_expected); - } - - #[test] - fn test_usage_count_of_unloaded_program() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - - let program = Pubkey::new_unique(); - let evict_to_pct = 2; - let cache_capacity_after_shrink = - Percentage::from(evict_to_pct).apply_to(crate::loaded_programs::MAX_LOADED_ENTRY_COUNT); - // Add enough programs to the cache to trigger 1 eviction after shrinking. - let num_total_programs = (cache_capacity_after_shrink + 1) as u64; - (0..num_total_programs).for_each(|i| { - let stats = ProgramStatistics { - uses: (i + 10).into(), - ..Default::default() - }; - let entry = new_test_entry_with_usage(i, i + 2, stats); - cache.assign_program(&env, program, i, entry); - }); - - cache.sort_and_unload(Percentage::from(evict_to_pct)); - - let num_unloaded = num_matching_entries(&cache, |program_type| { - matches!(program_type, ProgramCacheEntryType::Unloaded(_)) - }); - assert_eq!(num_unloaded, 1); - - cache - .get_flattened_entries_for_tests() - .iter() - .for_each(|(_key, program)| { - if matches!(program.program, ProgramCacheEntryType::Unloaded(_)) { - // Test that the usage counter is retained for the unloaded program - assert_eq!(program.stats.uses.load(Ordering::Relaxed), 10); - assert_eq!(program.deployment_slot, 0); - assert_eq!(program.effective_slot, 2); - } - }); - - // Replenish the program that was just unloaded. Use 0 as the usage counter. This should be - // updated with the usage counter from the unloaded program. - cache.assign_program( - &env, - program, - 0, - new_test_entry_with_usage(0, 2, ProgramStatistics::default()), - ); - - cache - .get_flattened_entries_for_tests() - .iter() - .for_each(|(_key, program)| { - if matches!(program.program, ProgramCacheEntryType::Unloaded(_)) - && program.deployment_slot == 0 - && program.effective_slot == 2 - { - // Test that the usage counter was correctly updated. - assert_eq!(program.stats.uses.load(Ordering::Relaxed), 10); - } - }); - } - - #[test] - fn test_fuzz_assign_program_order() { - use rand::prelude::SliceRandom; - const EXPECTED_ENTRIES: [(u64, u64); 7] = - [(1, 2), (5, 5), (5, 6), (5, 10), (9, 10), (10, 10), (3, 12)]; - let mut rng = rand::rng(); - let program_id = Pubkey::new_unique(); - let env = get_mock_program_runtime_environment(); - for _ in 0..1000 { - let mut entries = EXPECTED_ENTRIES.to_vec(); - entries.shuffle(&mut rng); - let mut cache = ProgramCache::::new(0); - for (deployment_slot, effective_slot) in entries { - let entry = Arc::new(ProgramCacheEntry { - program: new_loaded_entry(ProgramRuntimeEnvironment::from( - BuiltinProgram::new_mock(), - )), // Assign them different environments - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot, - effective_slot, - stats: Arc::default(), - latest_access_slot: AtomicU64::new(deployment_slot), - }); - assert!(!cache.assign_program(&env, program_id, deployment_slot, entry)); - } - for ((deployment_slot, effective_slot), entry) in EXPECTED_ENTRIES - .iter() - .zip(cache.get_slot_versions_for_tests(&program_id).iter()) - { - assert_eq!(entry.deployment_slot, *deployment_slot); - assert_eq!(entry.effective_slot, *effective_slot); - } - } - } - - #[test_matrix( - ( - ProgramCacheEntryType::Closed, - ProgramCacheEntryType::FailedVerification(get_mock_program_runtime_environment()), - new_loaded_entry(get_mock_program_runtime_environment()), - ), - ( - ProgramCacheEntryType::FailedVerification(get_mock_program_runtime_environment()), - ProgramCacheEntryType::Closed, - ProgramCacheEntryType::Unloaded(get_mock_program_runtime_environment()), - new_loaded_entry(get_mock_program_runtime_environment()), - ProgramCacheEntryType::Builtin(BuiltinProgram::new_mock()), - ) - )] - #[test_matrix( - ( - ProgramCacheEntryType::Unloaded(get_mock_program_runtime_environment()), - ), - ( - ProgramCacheEntryType::FailedVerification(get_mock_program_runtime_environment()), - ProgramCacheEntryType::Closed, - ProgramCacheEntryType::Unloaded(get_mock_program_runtime_environment()), - ProgramCacheEntryType::Builtin(BuiltinProgram::new_mock()), - ) - )] - #[test_matrix( - (ProgramCacheEntryType::Builtin(BuiltinProgram::new_mock()),), - ( - ProgramCacheEntryType::FailedVerification(get_mock_program_runtime_environment()), - ProgramCacheEntryType::Closed, - ProgramCacheEntryType::Unloaded(get_mock_program_runtime_environment()), - new_loaded_entry(get_mock_program_runtime_environment()), - ) - )] - #[should_panic(expected = "Unexpected replacement of an entry")] - fn test_assign_program_failure(old: ProgramCacheEntryType, new: ProgramCacheEntryType) { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - let program_id = Pubkey::new_unique(); - assert!(!cache.assign_program( - &env, - program_id, - 10, - Arc::new(ProgramCacheEntry { - program: old, - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot: 10, - effective_slot: 11, - stats: Arc::default(), - latest_access_slot: AtomicU64::default(), - }), - )); - cache.assign_program( - &env, - program_id, - 10, - Arc::new(ProgramCacheEntry { - program: new, - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot: 10, - effective_slot: 11, - stats: Arc::default(), - latest_access_slot: AtomicU64::default(), - }), - ); - } - - #[test_case( - ProgramCacheEntryType::Unloaded(ProgramRuntimeEnvironment::from( - BuiltinProgram::new_mock() - )), - new_loaded_entry(get_mock_program_runtime_environment()) - )] - #[test_case( - ProgramCacheEntryType::Builtin(BuiltinProgram::new_mock()), - ProgramCacheEntryType::Builtin(BuiltinProgram::new_mock()) - )] - fn test_assign_program_success(old: ProgramCacheEntryType, new: ProgramCacheEntryType) { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - let program_id = Pubkey::new_unique(); - assert!(!cache.assign_program( - &env, - program_id, - 10, - Arc::new(ProgramCacheEntry { - program: old, - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot: 10, - effective_slot: 11, - stats: Arc::default(), - latest_access_slot: AtomicU64::default(), - }), - )); - assert!(!cache.assign_program( - &env, - program_id, - 10, - Arc::new(ProgramCacheEntry { - program: new, - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot: 10, - effective_slot: 11, - stats: Arc::default(), - latest_access_slot: AtomicU64::default(), - }), - )); - } - - #[test] - fn test_assign_program_removes_entries_in_same_slot() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - let program_id = Pubkey::new_unique(); - let closed_other_slot = Arc::new(ProgramCacheEntry { - program: ProgramCacheEntryType::Closed, - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot: 9, - effective_slot: 9, - stats: Arc::default(), - latest_access_slot: AtomicU64::default(), - }); - let closed_current_slot = Arc::new(ProgramCacheEntry { - program: ProgramCacheEntryType::Closed, - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot: 10, - effective_slot: 10, - stats: Arc::default(), - latest_access_slot: AtomicU64::default(), - }); - let loaded_entry_current_env = Arc::new(ProgramCacheEntry { - program: ProgramCacheEntryType::Unloaded(get_mock_program_runtime_environment()), - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot: 10, - effective_slot: 11, - stats: Arc::default(), - latest_access_slot: AtomicU64::default(), - }); - let loaded_entry_upcoming_env = Arc::new(ProgramCacheEntry { - program: ProgramCacheEntryType::Unloaded(ProgramRuntimeEnvironment::from( - BuiltinProgram::new_mock(), - )), - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot: 10, - effective_slot: 11, - stats: Arc::default(), - latest_access_slot: AtomicU64::default(), - }); - assert!(!cache.assign_program(&env, program_id, 9, closed_other_slot.clone())); - assert!(!cache.assign_program(&env, program_id, 10, closed_current_slot)); - assert!(!cache.assign_program(&env, program_id, 10, loaded_entry_upcoming_env.clone())); - assert!(!cache.assign_program(&env, program_id, 10, loaded_entry_current_env.clone())); - // Only the conflicting entry in the same slot which does not have a different environment is removed - assert_eq!( - cache.get_slot_versions_for_tests(&program_id), - &[ - closed_other_slot, - loaded_entry_current_env, - loaded_entry_upcoming_env - ] - ); - } - - #[test] - fn test_tombstone() { - let env = get_mock_program_runtime_environment(); - let tombstone = ProgramCacheEntry::new_tombstone( - 0, - ProgramCacheEntryOwner::LoaderV2, - ProgramCacheEntryType::FailedVerification(env.clone()), - ); - assert_matches!( - tombstone.program, - ProgramCacheEntryType::FailedVerification(_) - ); - assert!(tombstone.is_tombstone()); - assert_eq!(tombstone.deployment_slot, 0); - assert_eq!(tombstone.effective_slot, 0); - - let tombstone = ProgramCacheEntry::new_tombstone( - 100, - ProgramCacheEntryOwner::LoaderV2, - ProgramCacheEntryType::Closed, - ); - assert_matches!(tombstone.program, ProgramCacheEntryType::Closed); - assert!(tombstone.is_tombstone()); - assert_eq!(tombstone.deployment_slot, 100); - assert_eq!(tombstone.effective_slot, 100); - - let mut cache = ProgramCache::::new(0); - let program1 = Pubkey::new_unique(); - let tombstone = set_tombstone( - &mut cache, - program1, - 10, - ProgramCacheEntryType::FailedVerification(env.clone()), - ); - let slot_versions = cache.get_slot_versions_for_tests(&program1); - assert_eq!(slot_versions.len(), 1); - assert!(slot_versions.first().unwrap().is_tombstone()); - assert_eq!(tombstone.deployment_slot, 10); - assert_eq!(tombstone.effective_slot, 10); - - // Add a program at slot 50, and a tombstone for the program at slot 60 - let program2 = Pubkey::new_unique(); - cache.assign_program(&env, program2, 50, new_test_builtin_entry(50, 51)); - let slot_versions = cache.get_slot_versions_for_tests(&program2); - assert_eq!(slot_versions.len(), 1); - assert!(!slot_versions.first().unwrap().is_tombstone()); - - let tombstone = set_tombstone( - &mut cache, - program2, - 60, - ProgramCacheEntryType::FailedVerification(env), - ); - let slot_versions = cache.get_slot_versions_for_tests(&program2); - assert_eq!(slot_versions.len(), 2); - assert!(!slot_versions.first().unwrap().is_tombstone()); - assert!(slot_versions.get(1).unwrap().is_tombstone()); - assert!(tombstone.is_tombstone()); - assert_eq!(tombstone.deployment_slot, 60); - assert_eq!(tombstone.effective_slot, 60); - } - - struct TestForkGraph { - relation: BlockRelation, - } - impl ForkGraph for TestForkGraph { - fn relationship(&self, _a: Slot, _b: Slot) -> BlockRelation { - self.relation - } - } - - #[test] - fn test_prune_empty() { - let mut cache = ProgramCache::::new(0); - let fork_graph = Arc::new(RwLock::new(TestForkGraph { - relation: BlockRelation::Unrelated, - })); - - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - cache.prune(0, None, &fork_graph.read().unwrap()); - assert!(cache.get_flattened_entries_for_tests().is_empty()); - - cache.prune(10, None, &fork_graph.read().unwrap()); - assert!(cache.get_flattened_entries_for_tests().is_empty()); - - let mut cache = ProgramCache::::new(0); - let fork_graph = Arc::new(RwLock::new(TestForkGraph { - relation: BlockRelation::Ancestor, - })); - - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - cache.prune(0, None, &fork_graph.read().unwrap()); - assert!(cache.get_flattened_entries_for_tests().is_empty()); - - cache.prune(10, None, &fork_graph.read().unwrap()); - assert!(cache.get_flattened_entries_for_tests().is_empty()); - - let mut cache = ProgramCache::::new(0); - let fork_graph = Arc::new(RwLock::new(TestForkGraph { - relation: BlockRelation::Descendant, - })); - - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - cache.prune(0, None, &fork_graph.read().unwrap()); - assert!(cache.get_flattened_entries_for_tests().is_empty()); - - cache.prune(10, None, &fork_graph.read().unwrap()); - assert!(cache.get_flattened_entries_for_tests().is_empty()); - - let mut cache = ProgramCache::::new(0); - let fork_graph = Arc::new(RwLock::new(TestForkGraph { - relation: BlockRelation::Unknown, - })); - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - cache.prune(0, None, &fork_graph.read().unwrap()); - assert!(cache.get_flattened_entries_for_tests().is_empty()); - - cache.prune(10, None, &fork_graph.read().unwrap()); - assert!(cache.get_flattened_entries_for_tests().is_empty()); - } - - #[test] - fn test_prune_different_env() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - - let fork_graph = Arc::new(RwLock::new(TestForkGraph { - relation: BlockRelation::Ancestor, - })); - - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - let program1 = Pubkey::new_unique(); - cache.assign_program(&env, program1, 10, new_test_entry(10, 10)); - let new_env = ProgramRuntimeEnvironment::from(BuiltinProgram::new_mock()); - let upcoming_environment = Some(new_env.clone()); - let updated_program = Arc::new(ProgramCacheEntry { - program: new_loaded_entry(new_env.clone()), - deployment_slot: 20, - effective_slot: 20, - ..Default::default() - }); - cache.assign_program( - &env, - program1, - updated_program.deployment_slot, - updated_program.clone(), - ); - - // Test that there are 2 entries for the program - assert_eq!(cache.get_slot_versions_for_tests(&program1).len(), 2); - - cache.prune(21, None, &fork_graph.read().unwrap()); - - // Test that prune didn't remove the entry, since environments are different. - assert_eq!(cache.get_slot_versions_for_tests(&program1).len(), 2); - - cache.prune(22, upcoming_environment, &fork_graph.read().unwrap()); - - // Test that prune removed 1 entry, since epoch changed - assert_eq!(cache.get_slot_versions_for_tests(&program1).len(), 1); - - let entry = cache - .get_slot_versions_for_tests(&program1) - .first() - .expect("Failed to get the program") - .clone(); - // Test that the correct entry remains in the cache - assert_eq!(entry, updated_program); - } - - #[derive(Default)] - struct TestForkGraphSpecific { - forks: Vec>, - } - - impl TestForkGraphSpecific { - fn insert_fork(&mut self, fork: &[Slot]) { - let mut fork = fork.to_vec(); - fork.sort(); - self.forks.push(fork) - } - } - - impl ForkGraph for TestForkGraphSpecific { - fn relationship(&self, a: Slot, b: Slot) -> BlockRelation { - match self.forks.iter().try_for_each(|fork| { - let relation = fork - .iter() - .position(|x| *x == a) - .and_then(|a_pos| { - fork.iter().position(|x| *x == b).and_then(|b_pos| { - (a_pos == b_pos) - .then_some(BlockRelation::Equal) - .or_else(|| (a_pos < b_pos).then_some(BlockRelation::Ancestor)) - .or(Some(BlockRelation::Descendant)) - }) - }) - .unwrap_or(BlockRelation::Unrelated); - - if relation != BlockRelation::Unrelated { - return ControlFlow::Break(relation); - } - - ControlFlow::Continue(()) - }) { - ControlFlow::Break(relation) => relation, - _ => BlockRelation::Unrelated, - } - } - } - - fn get_entries_to_load( - cache: &ProgramCache, - loading_slot: Slot, - keys: &[Pubkey], - ) -> Vec<(Pubkey, ProgramCacheMatchCriteria, Slot)> { - let fork_graph = cache.fork_graph.as_ref().unwrap().upgrade().unwrap(); - let locked_fork_graph = fork_graph.read().unwrap(); - let entries = cache.get_flattened_entries_for_tests(); - keys.iter() - .filter_map(|key| { - entries - .iter() - .rev() - .find(|(program_id, entry)| { - program_id == key - && matches!( - locked_fork_graph.relationship(entry.deployment_slot, loading_slot), - BlockRelation::Equal | BlockRelation::Ancestor, - ) - }) - .map(|(program_id, entry)| { - ( - *program_id, - ProgramCacheMatchCriteria::NoCriteria, - entry.deployment_slot, - ) - }) - }) - .collect() - } - - fn match_slot( - extracted: &ProgramCacheForTxBatch, - program: &Pubkey, - deployment_slot: Slot, - working_slot: Slot, - ) -> bool { - assert_eq!(extracted.slot, working_slot); - extracted - .entries - .get(program) - .map(|entry| entry.deployment_slot == deployment_slot) - .unwrap_or(false) - } - - fn match_missing( - missing: &[(Pubkey, ProgramCacheMatchCriteria, Slot)], - program: &Pubkey, - expected_result: bool, - ) -> bool { - missing.iter().any(|(key, _, _)| key == program) == expected_result - } - #[test] - fn test_fork_extract_and_prune() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - - // Fork graph created for the test - // 0 - // / \ - // 10 5 - // | | - // 20 11 - // | | \ - // 22 15 25 - // | | - // 16 27 - // | - // 19 - // | - // 23 - - let mut fork_graph = TestForkGraphSpecific::default(); - fork_graph.insert_fork(&[0, 10, 20, 22]); - fork_graph.insert_fork(&[0, 5, 11, 15, 16, 18, 19, 21, 23]); - fork_graph.insert_fork(&[0, 5, 11, 25, 27]); - - let fork_graph = Arc::new(RwLock::new(fork_graph)); - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - let program1 = Pubkey::new_unique(); - cache.assign_program(&env, program1, 0, new_test_entry(0, 1)); - cache.assign_program(&env, program1, 10, new_test_entry(10, 11)); - cache.assign_program(&env, program1, 20, new_test_entry(20, 21)); - - let program2 = Pubkey::new_unique(); - cache.assign_program(&env, program2, 5, new_test_entry(5, 6)); - cache.assign_program( - &env, - program2, - 11, - new_test_entry(11, 11 + DELAY_VISIBILITY_SLOT_OFFSET), - ); - - let program3 = Pubkey::new_unique(); - cache.assign_program(&env, program3, 25, new_test_entry(25, 26)); - - let program4 = Pubkey::new_unique(); - cache.assign_program(&env, program4, 0, new_test_entry(0, 1)); - cache.assign_program(&env, program4, 5, new_test_entry(5, 6)); - // The following is a special case, where effective slot is 3 slots in the future - cache.assign_program( - &env, - program4, - 15, - new_test_entry(15, 15 + DELAY_VISIBILITY_SLOT_OFFSET), - ); - - // Current fork graph - // 0 - // / \ - // 10 5 - // | | - // 20 11 - // | | \ - // 22 15 25 - // | | - // 16 27 - // | - // 19 - // | - // 23 - - // Testing fork 0 - 10 - 20 - 22 with current slot at 22 - let mut missing = - get_entries_to_load(&cache, 22, &[program1, program2, program3, program4]); - assert!(match_missing(&missing, &program2, false)); - assert!(match_missing(&missing, &program3, false)); - let mut extracted = ProgramCacheForTxBatch::new(22); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 20, 22)); - assert!(match_slot(&extracted, &program4, 0, 22)); - - // Testing fork 0 - 5 - 11 - 15 - 16 with current slot at 15 - let mut missing = - get_entries_to_load(&cache, 15, &[program1, program2, program3, program4]); - assert!(match_missing(&missing, &program3, false)); - let mut extracted = ProgramCacheForTxBatch::new(15); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 15)); - assert!(match_slot(&extracted, &program2, 11, 15)); - // The effective slot of program4 deployed in slot 15 is 19. So it should not be usable in slot 16. - // A delay visibility tombstone should be returned here. - let tombstone = extracted - .find(&program4) - .expect("Failed to find the tombstone"); - assert_matches!(tombstone.program, ProgramCacheEntryType::DelayVisibility); - assert_eq!(tombstone.deployment_slot, 15); - - // Testing the same fork above, but current slot is now 18 (equal to effective slot of program4). - let mut missing = - get_entries_to_load(&cache, 18, &[program1, program2, program3, program4]); - assert!(match_missing(&missing, &program3, false)); - let mut extracted = ProgramCacheForTxBatch::new(18); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 18)); - assert!(match_slot(&extracted, &program2, 11, 18)); - // The effective slot of program4 deployed in slot 15 is 18. So it should be usable in slot 18. - assert!(match_slot(&extracted, &program4, 15, 18)); - - // Testing the same fork above, but current slot is now 23 (future slot than effective slot of program4). - let mut missing = - get_entries_to_load(&cache, 23, &[program1, program2, program3, program4]); - assert!(match_missing(&missing, &program3, false)); - let mut extracted = ProgramCacheForTxBatch::new(23); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 23)); - assert!(match_slot(&extracted, &program2, 11, 23)); - // The effective slot of program4 deployed in slot 15 is 19. So it should be usable in slot 23. - assert!(match_slot(&extracted, &program4, 15, 23)); - - // Testing fork 0 - 5 - 11 - 15 - 16 with current slot at 11 - let mut missing = - get_entries_to_load(&cache, 11, &[program1, program2, program3, program4]); - assert!(match_missing(&missing, &program3, false)); - let mut extracted = ProgramCacheForTxBatch::new(11); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 11)); - // program2 was updated at slot 11, but is not effective till slot 12. The result should contain a tombstone. - let tombstone = extracted - .find(&program2) - .expect("Failed to find the tombstone"); - assert_matches!(tombstone.program, ProgramCacheEntryType::DelayVisibility); - assert_eq!(tombstone.deployment_slot, 11); - assert!(match_slot(&extracted, &program4, 5, 11)); - - cache.prune(5, None, &fork_graph.read().unwrap()); - - // Fork graph after pruning - // 0 - // | - // 5 - // | - // 11 - // | \ - // 15 25 - // | | - // 16 27 - // | - // 19 - // | - // 23 - - // Testing fork 11 - 15 - 16- 19 - 22 with root at 5 and current slot at 22 - let mut missing = - get_entries_to_load(&cache, 21, &[program1, program2, program3, program4]); - assert!(match_missing(&missing, &program3, false)); - let mut extracted = ProgramCacheForTxBatch::new(21); - cache.extract(&mut missing, &mut extracted, &env, true, true); - // Since the fork was pruned, we should not find the entry deployed at slot 20. - assert!(match_slot(&extracted, &program1, 0, 21)); - assert!(match_slot(&extracted, &program2, 11, 21)); - assert!(match_slot(&extracted, &program4, 15, 21)); - - // Testing fork 0 - 5 - 11 - 25 - 27 with current slot at 27 - let mut missing = - get_entries_to_load(&cache, 27, &[program1, program2, program3, program4]); - let mut extracted = ProgramCacheForTxBatch::new(27); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 27)); - assert!(match_slot(&extracted, &program2, 11, 27)); - assert!(match_slot(&extracted, &program3, 25, 27)); - assert!(match_slot(&extracted, &program4, 5, 27)); - - cache.prune(15, None, &fork_graph.read().unwrap()); - - // Fork graph after pruning - // 0 - // | - // 5 - // | - // 11 - // | - // 15 - // | - // 16 - // | - // 19 - // | - // 23 - - // Testing fork 16, 19, 23, with root at 15, current slot at 23 - let mut missing = - get_entries_to_load(&cache, 23, &[program1, program2, program3, program4]); - assert!(match_missing(&missing, &program3, false)); - let mut extracted = ProgramCacheForTxBatch::new(23); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 23)); - assert!(match_slot(&extracted, &program2, 11, 23)); - assert!(match_slot(&extracted, &program4, 15, 23)); - } - - #[test] - fn test_extract_using_deployment_slot() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - - // Fork graph created for the test - // 0 - // / \ - // 10 5 - // | | - // 20 11 - // | | \ - // 22 15 25 - // | | - // 16 27 - // | - // 19 - // | - // 23 - - let mut fork_graph = TestForkGraphSpecific::default(); - fork_graph.insert_fork(&[0, 10, 20, 22]); - fork_graph.insert_fork(&[0, 5, 11, 12, 15, 16, 18, 19, 21, 23]); - fork_graph.insert_fork(&[0, 5, 11, 25, 27]); - - let fork_graph = Arc::new(RwLock::new(fork_graph)); - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - let program1 = Pubkey::new_unique(); - cache.assign_program(&env, program1, 0, new_test_entry(0, 1)); - cache.assign_program(&env, program1, 20, new_test_entry(20, 21)); - - let program2 = Pubkey::new_unique(); - cache.assign_program(&env, program2, 5, new_test_entry(5, 6)); - cache.assign_program(&env, program2, 11, new_test_entry(11, 12)); - - let program3 = Pubkey::new_unique(); - cache.assign_program(&env, program3, 25, new_test_entry(25, 26)); - - // Testing fork 0 - 5 - 11 - 15 - 16 - 19 - 21 - 23 with current slot at 19 - let mut missing = get_entries_to_load(&cache, 12, &[program1, program2, program3]); - assert!(match_missing(&missing, &program3, false)); - let mut extracted = ProgramCacheForTxBatch::new(12); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 12)); - assert!(match_slot(&extracted, &program2, 11, 12)); - - // Test the same fork, but request the program modified at a later slot than what's in the cache. - let mut missing = get_entries_to_load(&cache, 12, &[program1, program2, program3]); - missing.get_mut(0).unwrap().1 = ProgramCacheMatchCriteria::DeployedOnOrAfterSlot(5); - missing.get_mut(1).unwrap().1 = ProgramCacheMatchCriteria::DeployedOnOrAfterSlot(5); - assert!(match_missing(&missing, &program3, false)); - let mut extracted = ProgramCacheForTxBatch::new(12); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_missing(&missing, &program1, true)); - assert!(match_slot(&extracted, &program2, 11, 12)); - } - - #[test] - fn test_extract_unloaded() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - - // Fork graph created for the test - // 0 - // / \ - // 10 5 - // | | - // 20 11 - // | | \ - // 22 15 25 - // | | - // 16 27 - // | - // 19 - // | - // 23 - - let mut fork_graph = TestForkGraphSpecific::default(); - fork_graph.insert_fork(&[0, 10, 20, 22]); - fork_graph.insert_fork(&[0, 5, 11, 15, 16, 19, 21, 23]); - fork_graph.insert_fork(&[0, 5, 11, 25, 27]); - - let fork_graph = Arc::new(RwLock::new(fork_graph)); - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - let program1 = Pubkey::new_unique(); - cache.assign_program(&env, program1, 0, new_test_entry(0, 1)); - cache.assign_program(&env, program1, 20, new_test_entry(20, 21)); - - let program2 = Pubkey::new_unique(); - cache.assign_program(&env, program2, 5, new_test_entry(5, 6)); - cache.assign_program(&env, program2, 11, new_test_entry(11, 12)); - - let program3 = Pubkey::new_unique(); - // Insert an unloaded program with correct/cache's environment at slot 25 - let _ = insert_unloaded_entry(&mut cache, program3, 25); - - // Insert another unloaded program with a different environment at slot 20 - // Since this entry's environment won't match cache's environment, looking up this - // entry should return missing instead of unloaded entry. - cache.assign_program( - &env, - program3, - 20, - Arc::new( - new_test_entry(20, 21) - .to_unloaded() - .expect("Failed to create unloaded program"), - ), - ); - - // Testing fork 0 - 5 - 11 - 15 - 16 - 19 - 21 - 23 with current slot at 19 - let mut missing = get_entries_to_load(&cache, 19, &[program1, program2, program3]); - assert!(match_missing(&missing, &program3, false)); - let mut extracted = ProgramCacheForTxBatch::new(19); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 19)); - assert!(match_slot(&extracted, &program2, 11, 19)); - - // Testing fork 0 - 5 - 11 - 25 - 27 with current slot at 27 - let mut missing = get_entries_to_load(&cache, 27, &[program1, program2, program3]); - let mut extracted = ProgramCacheForTxBatch::new(27); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 27)); - assert!(match_slot(&extracted, &program2, 11, 27)); - assert!(match_missing(&missing, &program3, true)); - - // Testing fork 0 - 10 - 20 - 22 with current slot at 22 - let mut missing = get_entries_to_load(&cache, 22, &[program1, program2, program3]); - assert!(match_missing(&missing, &program2, false)); - let mut extracted = ProgramCacheForTxBatch::new(22); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 20, 22)); - assert!(match_missing(&missing, &program3, true)); - } - - #[test] - fn test_extract_different_environment() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - let other_env = ProgramRuntimeEnvironment::from(BuiltinProgram::new_mock()); - - // Fork graph created for the test - // 0 - // | - // 10 - // | - // 20 - // | - // 22 - - let mut fork_graph = TestForkGraphSpecific::default(); - fork_graph.insert_fork(&[0, 10, 20, 22]); - - let fork_graph = Arc::new(RwLock::new(fork_graph)); - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - let program1 = Pubkey::new_unique(); - cache.assign_program( - &env, - program1, - 10, - Arc::new(ProgramCacheEntry::new_tombstone( - 10, - ProgramCacheEntryOwner::LoaderV3, - ProgramCacheEntryType::Closed, - )), - ); - cache.assign_program(&env, program1, 20, new_test_entry(20, 21)); - - // Testing fork 0 - 10 - 20 - 22 with current slot at 22 - let mut missing = get_entries_to_load(&cache, 22, &[program1]); - let mut extracted = ProgramCacheForTxBatch::new(22); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 20, 22)); - - // Looking for a different environment - let mut missing = get_entries_to_load(&cache, 22, &[program1]); - let mut extracted = ProgramCacheForTxBatch::new(22); - cache.extract(&mut missing, &mut extracted, &other_env, true, true); - assert!(match_missing(&missing, &program1, true)); - } - - #[test] - fn test_extract_nonexistent() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - let fork_graph = TestForkGraphSpecific::default(); - let fork_graph = Arc::new(RwLock::new(fork_graph)); - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - let program1 = Pubkey::new_unique(); - let mut missing = vec![(program1, ProgramCacheMatchCriteria::NoCriteria, 0)]; - let mut extracted = ProgramCacheForTxBatch::new(0); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_missing(&missing, &program1, true)); - } - - #[test] - fn test_unloaded() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - for program_cache_entry_type in [ - ProgramCacheEntryType::FailedVerification(get_mock_program_runtime_environment()), - ProgramCacheEntryType::Closed, - ProgramCacheEntryType::Unloaded(get_mock_program_runtime_environment()), - ProgramCacheEntryType::Builtin(BuiltinProgram::new_mock()), - ] { - let entry = Arc::new(ProgramCacheEntry { - program: program_cache_entry_type, - account_owner: ProgramCacheEntryOwner::LoaderV2, - account_size: 0, - deployment_slot: 0, - effective_slot: 0, - stats: Arc::default(), - latest_access_slot: AtomicU64::default(), - }); - assert!(entry.to_unloaded().is_none()); - - // Check that unload_program_entry() does nothing for this entry - let program_id = Pubkey::new_unique(); - cache.assign_program(&env, program_id, entry.deployment_slot, entry.clone()); - cache.unload_program_entry(program_id, entry.deployment_slot, &entry); - assert_eq!(cache.get_slot_versions_for_tests(&program_id).len(), 1); - assert!(cache.stats.evictions.is_empty()); - } - - let stats = ProgramStatistics { - uses: 3.into(), - ..Default::default() - }; - let entry = new_test_entry_with_usage(1, 2, stats); - let unloaded_entry = entry.to_unloaded().unwrap(); - assert_eq!(unloaded_entry.deployment_slot, 1); - assert_eq!(unloaded_entry.effective_slot, 2); - assert_eq!(unloaded_entry.latest_access_slot.load(Ordering::Relaxed), 1); - assert_eq!(unloaded_entry.stats.uses.load(Ordering::Relaxed), 3); - - // Check that unload_program_entry() does its work - let program_id = Pubkey::new_unique(); - cache.assign_program(&env, program_id, entry.deployment_slot, entry.clone()); - cache.unload_program_entry(program_id, entry.deployment_slot, &entry); - assert!(cache.stats.evictions.contains_key(&program_id)); - } - - #[test] - fn test_fork_prune_find_first_ancestor() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - - // Fork graph created for the test - // 0 - // / \ - // 10 5 - // | - // 20 - - // Deploy program on slot 0, and slot 5. - // Prune the fork that has slot 5. The cache should still have the program - // deployed at slot 0. - let mut fork_graph = TestForkGraphSpecific::default(); - fork_graph.insert_fork(&[0, 10, 20]); - fork_graph.insert_fork(&[0, 5]); - let fork_graph = Arc::new(RwLock::new(fork_graph)); - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - let program1 = Pubkey::new_unique(); - cache.assign_program(&env, program1, 0, new_test_entry(0, 1)); - cache.assign_program(&env, program1, 5, new_test_entry(5, 6)); - - cache.prune(10, None, &fork_graph.read().unwrap()); - - let mut missing = get_entries_to_load(&cache, 20, &[program1]); - let mut extracted = ProgramCacheForTxBatch::new(20); - cache.extract(&mut missing, &mut extracted, &env, true, true); - - // The cache should have the program deployed at slot 0 - assert_eq!( - extracted - .find(&program1) - .expect("Did not find the program") - .deployment_slot, - 0 - ); - } - - #[test] - fn test_prune_by_deployment_slot() { - let mut cache = ProgramCache::::new(0); - let env = get_mock_program_runtime_environment(); - - // Fork graph created for the test - // 0 - // / \ - // 10 5 - // | - // 20 - - // Deploy program on slot 0, and slot 5. - // Prune the fork that has slot 5. The cache should still have the program - // deployed at slot 0. - let mut fork_graph = TestForkGraphSpecific::default(); - fork_graph.insert_fork(&[0, 10, 20]); - fork_graph.insert_fork(&[0, 5, 6]); - let fork_graph = Arc::new(RwLock::new(fork_graph)); - cache.set_fork_graph(Arc::downgrade(&fork_graph)); - - let program1 = Pubkey::new_unique(); - cache.assign_program(&env, program1, 0, new_test_entry(0, 1)); - cache.assign_program(&env, program1, 5, new_test_entry(5, 6)); - - let program2 = Pubkey::new_unique(); - cache.assign_program(&env, program2, 10, new_test_entry(10, 11)); - - let mut missing = get_entries_to_load(&cache, 20, &[program1, program2]); - let mut extracted = ProgramCacheForTxBatch::new(20); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 20)); - assert!(match_slot(&extracted, &program2, 10, 20)); - - let mut missing = get_entries_to_load(&cache, 6, &[program1, program2]); - assert!(match_missing(&missing, &program2, false)); - let mut extracted = ProgramCacheForTxBatch::new(6); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 5, 6)); - - // Pruning slot 5 will remove program1 entry deployed at slot 5. - // On fork chaining from slot 5, the entry deployed at slot 0 will become visible. - cache.prune_by_deployment_slot(5); - - let mut missing = get_entries_to_load(&cache, 20, &[program1, program2]); - let mut extracted = ProgramCacheForTxBatch::new(20); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 20)); - assert!(match_slot(&extracted, &program2, 10, 20)); - - let mut missing = get_entries_to_load(&cache, 6, &[program1, program2]); - assert!(match_missing(&missing, &program2, false)); - let mut extracted = ProgramCacheForTxBatch::new(6); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 6)); - - // Pruning slot 10 will remove program2 entry deployed at slot 10. - // As there is no other entry for program2, extract() will return it as missing. - cache.prune_by_deployment_slot(10); - - let mut missing = get_entries_to_load(&cache, 20, &[program1, program2]); - assert!(match_missing(&missing, &program2, false)); - let mut extracted = ProgramCacheForTxBatch::new(20); - cache.extract(&mut missing, &mut extracted, &env, true, true); - assert!(match_slot(&extracted, &program1, 0, 20)); - } - - #[test] - fn test_usable_entries_for_slot() { - ProgramCache::::new(0); - let tombstone = Arc::new(ProgramCacheEntry::new_tombstone( - 0, - ProgramCacheEntryOwner::LoaderV2, - ProgramCacheEntryType::Closed, - )); - - assert!(ProgramCache::::matches_criteria( - &tombstone, - &ProgramCacheMatchCriteria::NoCriteria - )); - - assert!(ProgramCache::::matches_criteria( - &tombstone, - &ProgramCacheMatchCriteria::Tombstone - )); - - assert!(ProgramCache::::matches_criteria( - &tombstone, - &ProgramCacheMatchCriteria::DeployedOnOrAfterSlot(0) - )); - - assert!(!ProgramCache::::matches_criteria( - &tombstone, - &ProgramCacheMatchCriteria::DeployedOnOrAfterSlot(1) - )); - - let program = new_test_entry(0, 1); - - assert!(ProgramCache::::matches_criteria( - &program, - &ProgramCacheMatchCriteria::NoCriteria - )); - - assert!(!ProgramCache::::matches_criteria( - &program, - &ProgramCacheMatchCriteria::Tombstone - )); - - assert!(ProgramCache::::matches_criteria( - &program, - &ProgramCacheMatchCriteria::DeployedOnOrAfterSlot(0) - )); - - assert!(!ProgramCache::::matches_criteria( - &program, - &ProgramCacheMatchCriteria::DeployedOnOrAfterSlot(1) - )); - - let program = Arc::new(new_test_entry_with_usage( - 0, - 1, - ProgramStatistics::default(), - )); - - assert!(ProgramCache::::matches_criteria( - &program, - &ProgramCacheMatchCriteria::NoCriteria - )); - - assert!(!ProgramCache::::matches_criteria( - &program, - &ProgramCacheMatchCriteria::Tombstone - )); - - assert!(ProgramCache::::matches_criteria( - &program, - &ProgramCacheMatchCriteria::DeployedOnOrAfterSlot(0) - )); - - assert!(!ProgramCache::::matches_criteria( - &program, - &ProgramCacheMatchCriteria::DeployedOnOrAfterSlot(1) - )); + fn assign_program_makes_entry_retrievable() { + let cache = ProgramCache::default(); + let key = Pubkey::new_unique(); + let entry = test_entry(); + cache.assign_program(key, entry.clone()); + + let fetched = cache.get(&key); + assert!(fetched.is_some()); + assert!(Arc::ptr_eq(&fetched.unwrap(), &entry)); } } diff --git a/solana/program-runtime/src/loading_task.rs b/solana/program-runtime/src/loading_task.rs deleted file mode 100644 index fe9ed92..0000000 --- a/solana/program-runtime/src/loading_task.rs +++ /dev/null @@ -1,49 +0,0 @@ -use solana_svm_type_overrides::sync::{Condvar, Mutex}; - -#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] -pub struct LoadingTaskCookie(u64); - -impl LoadingTaskCookie { - fn new() -> Self { - Self(0) - } - - fn update(&mut self) { - let LoadingTaskCookie(cookie) = self; - *cookie = cookie.wrapping_add(1); - } -} - -/// Suspends the thread in case no cooprative loading task was assigned -#[derive(Debug, Default)] -pub struct LoadingTaskWaiter { - cookie: Mutex, - cond: Condvar, -} - -impl LoadingTaskWaiter { - pub fn new() -> Self { - Self { - cookie: Mutex::new(LoadingTaskCookie::new()), - cond: Condvar::new(), - } - } - - pub fn cookie(&self) -> LoadingTaskCookie { - *self.cookie.lock().unwrap() - } - - pub fn notify(&self) { - let mut cookie = self.cookie.lock().unwrap(); - cookie.update(); - self.cond.notify_all(); - } - - pub fn wait(&self, cookie: LoadingTaskCookie) -> LoadingTaskCookie { - let cookie_guard = self.cookie.lock().unwrap(); - *self - .cond - .wait_while(cookie_guard, |current_cookie| *current_cookie == cookie) - .unwrap() - } -} diff --git a/solana/program-runtime/src/mem_pool.rs b/solana/program-runtime/src/mem_pool.rs index 9907baa..519d610 100644 --- a/solana/program-runtime/src/mem_pool.rs +++ b/solana/program-runtime/src/mem_pool.rs @@ -1,13 +1,10 @@ use { crate::execution_budget::{ - MAX_CALL_DEPTH, MAX_HEAP_FRAME_BYTES, MAX_INSTRUCTION_STACK_DEPTH_SIMD_0268, - MIN_HEAP_FRAME_BYTES, - }, - solana_sbpf::{aligned_memory::AlignedMemory, ebpf::HOST_ALIGN, vm::CallFrame}, - std::{ - array, - ops::{Deref, DerefMut}, + MAX_CALL_DEPTH, MAX_HEAP_FRAME_BYTES, MAX_INSTRUCTION_STACK_DEPTH, MIN_HEAP_FRAME_BYTES, + STACK_FRAME_SIZE, }, + solana_sbpf::{aligned_memory::AlignedMemory, ebpf::HOST_ALIGN}, + std::array, }; trait Reset { @@ -36,9 +33,7 @@ impl Pool { return None; } self.next_empty = self.next_empty.saturating_sub(1); - self.items - .get_mut(self.next_empty) - .and_then(|item| item.take()) + self.items.get_mut(self.next_empty).and_then(|item| item.take()) } fn put(&mut self, mut value: T) -> bool { @@ -60,79 +55,47 @@ impl Reset for AlignedMemory<{ HOST_ALIGN }> { } } -pub struct CallFrameBuffer(Box<[CallFrame; MAX_CALL_DEPTH]>); - -impl Default for CallFrameBuffer { - fn default() -> Self { - let mut mem = Box::<[CallFrame; MAX_CALL_DEPTH]>::new_uninit(); - let ptr = mem.as_mut_ptr().cast::(); - for i in 0..MAX_CALL_DEPTH { - unsafe { ptr.add(i).write(CallFrame::default()) } - } - Self(unsafe { mem.assume_init() }) - } -} - -impl Reset for CallFrameBuffer { - fn reset(&mut self) { - self.fill(CallFrame::default()) - } -} - -impl Deref for CallFrameBuffer { - type Target = [CallFrame]; - - fn deref(&self) -> &Self::Target { - self.0.as_slice() - } -} - -impl DerefMut for CallFrameBuffer { - fn deref_mut(&mut self) -> &mut Self::Target { - self.0.as_mut_slice() - } -} - +/// Fixed-size reusable stack and heap buffers for SBF VMs. pub struct VmMemoryPool { - stack: Pool, MAX_INSTRUCTION_STACK_DEPTH_SIMD_0268>, - heap: Pool, MAX_INSTRUCTION_STACK_DEPTH_SIMD_0268>, - call_frame: Pool, + stack: Pool, MAX_INSTRUCTION_STACK_DEPTH>, + heap: Pool, MAX_INSTRUCTION_STACK_DEPTH>, } impl VmMemoryPool { + /// Allocates a pool sized for the maximum instruction stack depth. pub fn new() -> Self { Self { stack: Pool::new(array::from_fn(|_| { - #[allow(clippy::arithmetic_side_effects)] - AlignedMemory::zero_filled(solana_sbpf::vm::get_stack_frame_size() * MAX_CALL_DEPTH) + AlignedMemory::zero_filled(STACK_FRAME_SIZE * MAX_CALL_DEPTH) })), heap: Pool::new(array::from_fn(|_| { AlignedMemory::zero_filled(MAX_HEAP_FRAME_BYTES as usize) })), - call_frame: Pool::new(array::from_fn(|_| CallFrameBuffer::default())), } } + /// Number of stack buffers managed by the pool. pub fn stack_len(&self) -> usize { self.stack.len() } + /// Number of heap buffers managed by the pool. pub fn heap_len(&self) -> usize { self.heap.len() } - #[allow(clippy::arithmetic_side_effects)] + /// Returns a zeroed stack buffer, allocating one if the pool is empty. pub fn get_stack(&mut self, size: usize) -> AlignedMemory<{ HOST_ALIGN }> { - debug_assert!(size == solana_sbpf::vm::get_stack_frame_size() * MAX_CALL_DEPTH); - self.stack - .get() - .unwrap_or_else(|| AlignedMemory::zero_filled(size)) + debug_assert!(size == STACK_FRAME_SIZE * MAX_CALL_DEPTH); + self.stack.get().unwrap_or_else(|| AlignedMemory::zero_filled(size)) } + /// Returns a stack buffer to the pool after clearing it. pub fn put_stack(&mut self, stack: AlignedMemory<{ HOST_ALIGN }>) -> bool { self.stack.put(stack) } + /// Returns a zeroed heap buffer, allocating one if the pool is empty. pub fn get_heap(&mut self, heap_size: u32) -> AlignedMemory<{ HOST_ALIGN }> { debug_assert!((MIN_HEAP_FRAME_BYTES..=MAX_HEAP_FRAME_BYTES).contains(&heap_size)); self.heap @@ -140,6 +103,7 @@ impl VmMemoryPool { .unwrap_or_else(|| AlignedMemory::zero_filled(MAX_HEAP_FRAME_BYTES as usize)) } + /// Returns a heap buffer to the pool after clearing it. pub fn put_heap(&mut self, heap: AlignedMemory<{ HOST_ALIGN }>) -> bool { let heap_size = heap.len(); debug_assert!( @@ -148,14 +112,6 @@ impl VmMemoryPool { ); self.heap.put(heap) } - - pub fn get_call_frames(&mut self) -> CallFrameBuffer { - self.call_frame.get().unwrap_or_default() - } - - pub fn put_call_frames(&mut self, call_frame: CallFrameBuffer) -> bool { - self.call_frame.put(call_frame) - } } impl Default for VmMemoryPool { diff --git a/solana/program-runtime/src/memory.rs b/solana/program-runtime/src/memory.rs index 508a1b5..725ccd5 100644 --- a/solana/program-runtime/src/memory.rs +++ b/solana/program-runtime/src/memory.rs @@ -7,11 +7,11 @@ use { }; /// Error types for memory translation operations. -#[derive(Debug, thiserror::Error, PartialEq, Eq, Clone)] +#[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum MemoryTranslationError { #[error("Unaligned pointer")] UnalignedPointer, - #[error("InvalidLength")] + #[error("Invalid length")] InvalidLength, } @@ -27,9 +27,7 @@ pub fn address_is_aligned(address: u64) -> bool { macro_rules! translate_inner { ($memory_mapping:expr, $map:ident, $access_type:expr, $vm_addr:expr, $len:expr $(,)?) => { Result::>::from( - $memory_mapping - .$map($access_type, $vm_addr, $len) - .map_err(|err| err.into()), + $memory_mapping.$map($access_type, $vm_addr, $len).map_err(|err| err.into()), ) }; } diff --git a/solana/program-runtime/src/memory_context.rs b/solana/program-runtime/src/memory_context.rs index 2a91946..b82d556 100644 --- a/solana/program-runtime/src/memory_context.rs +++ b/solana/program-runtime/src/memory_context.rs @@ -14,81 +14,40 @@ pub struct MemoryContexts { impl MemoryContexts { pub(crate) fn new() -> Self { - Self { - contexts: Vec::new(), - } + Self { contexts: Vec::new() } } - /// Set this instruction's [`MemoryContext`]. pub fn set_memory_context_abi_v1( &mut self, memory_context: MemoryContext, ) -> Result<(), InstructionError> { - *self - .contexts - .last_mut() - .ok_or(InstructionError::CallDepth)? = MemoryContextType::ABIv1(memory_context); + *self.contexts.last_mut().ok_or(InstructionError::CallDepth)? = + MemoryContextType::ABIv1(memory_context); Ok(()) } - /// Get current instruction's [`MemoryContext`] - pub fn memory_context_abi_v1(&self) -> Result<&MemoryContext, InstructionError> { - match self.contexts.last().ok_or(InstructionError::CallDepth)? { - MemoryContextType::ABIv1(ctx) => Ok(ctx), - MemoryContextType::Placeholder => Err(InstructionError::ProgramEnvironmentSetupFailure), - } - } - - /// Get current instruction's [`MemoryContext`] for mutable use. pub fn memory_context_mut_abi_v1(&mut self) -> Result<&mut MemoryContext, InstructionError> { - let context = self - .contexts - .last_mut() - .ok_or(InstructionError::CallDepth)?; - - match context { - MemoryContextType::ABIv1(ctx) => Ok(ctx), + match self.contexts.last_mut().ok_or(InstructionError::CallDepth)? { + MemoryContextType::ABIv1(context) => Ok(context), MemoryContextType::Placeholder => Err(InstructionError::ProgramEnvironmentSetupFailure), } } pub fn memory_mapping(&self) -> Result<&MemoryMapping, InstructionError> { - let mapping = match self.contexts.last().ok_or(InstructionError::CallDepth)? { - MemoryContextType::ABIv1(ctx) => &ctx.memory_mapping, - MemoryContextType::Placeholder => { - return Err(InstructionError::ProgramEnvironmentSetupFailure); - } - }; - - Ok(mapping) + match self.contexts.last().ok_or(InstructionError::CallDepth)? { + MemoryContextType::ABIv1(context) => Ok(&context.memory_mapping), + MemoryContextType::Placeholder => Err(InstructionError::ProgramEnvironmentSetupFailure), + } } pub fn memory_mapping_mut(&mut self) -> Result<&mut MemoryMapping, InstructionError> { - let mapping = match self - .contexts - .last_mut() - .ok_or(InstructionError::CallDepth)? - { - MemoryContextType::ABIv1(ctx) => &mut ctx.memory_mapping, - MemoryContextType::Placeholder => { - return Err(InstructionError::ProgramEnvironmentSetupFailure); - } - }; - - Ok(mapping) - } - - #[cfg(feature = "dev-context-only-utils")] - pub fn mock_set_mapping_abi_v1(&mut self, memory_mapping: MemoryMapping) { - self.contexts = vec![MemoryContextType::ABIv1(MemoryContext { - allocator: BpfAllocator::new(0), - accounts_metadata: vec![], - memory_mapping: Box::new(memory_mapping), - })]; + match self.contexts.last_mut().ok_or(InstructionError::CallDepth)? { + MemoryContextType::ABIv1(context) => Ok(&mut context.memory_mapping), + MemoryContextType::Placeholder => Err(InstructionError::ProgramEnvironmentSetupFailure), + } } pub fn push_placeholder(&mut self) { - // We are only pushing a placeholder to be configured later self.contexts.push(MemoryContextType::Placeholder); } @@ -97,8 +56,6 @@ impl MemoryContexts { } } -/// This structure contains metadata about the memory for each instruction under execution. -/// The BpfAllocator, accounts addresses in the guest and the memory mapping. pub struct MemoryContext { pub allocator: BpfAllocator, pub accounts_metadata: Vec, @@ -106,7 +63,6 @@ pub struct MemoryContext { } impl MemoryContext { - /// Creates a new memory context pub fn new( allocator: BpfAllocator, accounts_metadata: Vec, @@ -122,8 +78,6 @@ impl MemoryContext { #[derive(Debug, Clone)] pub struct SerializedAccountMetadata { - /// Address of the first byte of the serialized account record (the - /// `NON_DUP_MARKER`/duplicate-marker byte). pub vm_addr: u64, pub original_data_len: usize, pub vm_data_addr: u64, diff --git a/solana/program-runtime/src/program_cache_entry.rs b/solana/program-runtime/src/program_cache_entry.rs deleted file mode 100644 index 35764a7..0000000 --- a/solana/program-runtime/src/program_cache_entry.rs +++ /dev/null @@ -1,522 +0,0 @@ -#[cfg(feature = "metrics")] -use crate::program_metrics::LoadProgramMetrics; -use { - crate::{ - invoke_context::{BuiltinFunctionRegisterer, InvokeContext}, - loaded_programs::ProgramRuntimeEnvironment, - program_metrics::ProgramStatistics, - }, - solana_clock::Slot, - solana_pubkey::Pubkey, - solana_sbpf::{elf::Executable, program::BuiltinProgram, verifier::RequisiteVerifier}, - solana_sdk_ids::{ - bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable, loader_v4, native_loader, - }, - solana_svm_type_overrides::sync::{ - Arc, - atomic::{AtomicU64, Ordering}, - }, -}; - -pub const DELAY_VISIBILITY_SLOT_OFFSET: Slot = 1; - -/// The owner of a programs accounts, thus the loader of a program -#[derive(Default, Clone, Copy, PartialEq, Eq, Debug)] -pub enum ProgramCacheEntryOwner { - #[default] - NativeLoader, - LoaderV1, - LoaderV2, - LoaderV3, - LoaderV4, -} - -impl TryFrom<&Pubkey> for ProgramCacheEntryOwner { - type Error = (); - fn try_from(loader_key: &Pubkey) -> Result { - if native_loader::check_id(loader_key) { - Ok(ProgramCacheEntryOwner::NativeLoader) - } else if bpf_loader_deprecated::check_id(loader_key) { - Ok(ProgramCacheEntryOwner::LoaderV1) - } else if bpf_loader::check_id(loader_key) { - Ok(ProgramCacheEntryOwner::LoaderV2) - } else if bpf_loader_upgradeable::check_id(loader_key) { - Ok(ProgramCacheEntryOwner::LoaderV3) - } else if loader_v4::check_id(loader_key) { - Ok(ProgramCacheEntryOwner::LoaderV4) - } else { - Err(()) - } - } -} - -impl From for Pubkey { - fn from(program_cache_entry_owner: ProgramCacheEntryOwner) -> Self { - match program_cache_entry_owner { - ProgramCacheEntryOwner::NativeLoader => native_loader::id(), - ProgramCacheEntryOwner::LoaderV1 => bpf_loader_deprecated::id(), - ProgramCacheEntryOwner::LoaderV2 => bpf_loader::id(), - ProgramCacheEntryOwner::LoaderV3 => bpf_loader_upgradeable::id(), - ProgramCacheEntryOwner::LoaderV4 => loader_v4::id(), - } - } -} - -/* - The possible ProgramCacheEntryType transitions: - - DelayVisibility is special in that it is never stored in the cache. - It is only returned by ProgramCacheForTxBatch::find() when a Loaded entry - is encountered which is not effective yet. - - Builtin re/deployment: - - Empty => Builtin in TransactionBatchProcessor::add_builtin - - Builtin => Builtin in TransactionBatchProcessor::add_builtin - - Un/re/deployment (with delay and cooldown): - - Empty / Closed => Loaded in UpgradeableLoaderInstruction::DeployWithMaxDataLen - - Loaded / FailedVerification => Loaded in UpgradeableLoaderInstruction::Upgrade - - Loaded / FailedVerification => Closed in UpgradeableLoaderInstruction::Close - - Loader migration: - - Closed => Closed (in the same slot) - - FailedVerification => FailedVerification (with different account_owner) - - Loaded => Loaded (with different account_owner) - - Eviction and unloading (in the same slot): - - Unloaded => Loaded in ProgramCache::assign_program - - Loaded => Unloaded in ProgramCache::unload_program_entry - - At epoch boundary (when feature set and environment changes): - - Loaded => FailedVerification in Bank::_new_from_parent - - FailedVerification => Loaded in Bank::_new_from_parent - - Through pruning (when on orphan fork or overshadowed on the rooted fork): - - Closed / Unloaded / Loaded / Builtin => Empty in ProgramCache::prune -*/ - -/// Actual payload of [ProgramCacheEntry]. -#[derive(Default)] -pub enum ProgramCacheEntryType { - /// Tombstone for programs which currently do not pass the verifier but could if the feature set changed. - FailedVerification(ProgramRuntimeEnvironment), - /// Tombstone for programs that were either explicitly closed or never deployed. - /// - /// It's also used for accounts belonging to program loaders, that don't actually contain program code (e.g. buffer accounts for LoaderV3 programs). - #[default] - Closed, - /// Tombstone for programs which have recently been modified but the new version is not visible yet. - DelayVisibility, - /// Successfully verified but not currently compiled. - /// - /// It continues to track usage statistics even when the compiled executable of the program is evicted from memory. - Unloaded(ProgramRuntimeEnvironment), - /// Verified program. - /// - /// It may or may not be JIT compiled. - Loaded(Executable>), - /// A built-in program which is not stored on-chain but backed into and distributed with the validator - Builtin(BuiltinProgram>), -} - -impl std::fmt::Debug for ProgramCacheEntryType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct(match self { - ProgramCacheEntryType::FailedVerification(_) => { - "ProgramCacheEntryType::FailedVerification" - } - ProgramCacheEntryType::Closed => "ProgramCacheEntryType::Closed", - ProgramCacheEntryType::DelayVisibility => "ProgramCacheEntryType::DelayVisibility", - ProgramCacheEntryType::Unloaded(_) => "ProgramCacheEntryType::Unloaded", - ProgramCacheEntryType::Loaded(_) => "ProgramCacheEntryType::Loaded", - ProgramCacheEntryType::Builtin(_) => "ProgramCacheEntryType::Builtin", - }) - .finish() - } -} - -impl ProgramCacheEntryType { - /// Returns a reference to its environment if it has one - pub fn get_environment(&self) -> Option<&ProgramRuntimeEnvironment> { - match self { - ProgramCacheEntryType::Loaded(program) => { - Some(ProgramRuntimeEnvironment::from_ref(program.get_loader())) - } - ProgramCacheEntryType::FailedVerification(env) - | ProgramCacheEntryType::Unloaded(env) => Some(env), - _ => None, - } - } -} - -/// Holds a program version at a specific address and on a specific slot / fork. -/// -/// It contains the actual program in [ProgramCacheEntryType] and a bunch of meta-data. -#[derive(Debug, Default)] -pub struct ProgramCacheEntry { - /// The program of this entry - pub program: ProgramCacheEntryType, - /// The loader of this entry - pub account_owner: ProgramCacheEntryOwner, - /// Size of account that stores the program and program data - pub account_size: usize, - /// Slot in which the program was (re)deployed - pub deployment_slot: Slot, - /// Slot in which this entry will become active (can be in the future) - pub effective_slot: Slot, - /// How often this entry was used by a transaction - pub stats: Arc, - pub latest_access_slot: AtomicU64, -} - -impl PartialEq for ProgramCacheEntry { - fn eq(&self, other: &Self) -> bool { - self.effective_slot == other.effective_slot - && self.deployment_slot == other.deployment_slot - && self.is_tombstone() == other.is_tombstone() - } -} - -impl ProgramCacheEntry { - /// Creates a new user program - pub fn new( - loader_key: &Pubkey, - program_runtime_environment: ProgramRuntimeEnvironment, - deployment_slot: Slot, - effective_slot: Slot, - elf_bytes: &[u8], - account_size: usize, - #[cfg(feature = "metrics")] metrics: &mut LoadProgramMetrics, - ) -> Result> { - Self::new_internal( - loader_key, - program_runtime_environment, - deployment_slot, - effective_slot, - elf_bytes, - account_size, - #[cfg(feature = "metrics")] - metrics, - false, /* reloading */ - ) - } - - /// Reloads a user program, *without* running the verifier. - /// - /// # Safety - /// - /// This method is unsafe since it assumes that the program has already been verified. Should - /// only be called when the program was previously verified and loaded in the cache, but was - /// unloaded due to inactivity. It should also be checked that the `program_runtime_environment` - /// hasn't changed since it was unloaded. - pub unsafe fn reload( - loader_key: &Pubkey, - program_runtime_environment: ProgramRuntimeEnvironment, - deployment_slot: Slot, - effective_slot: Slot, - elf_bytes: &[u8], - account_size: usize, - #[cfg(feature = "metrics")] metrics: &mut LoadProgramMetrics, - ) -> Result> { - Self::new_internal( - loader_key, - program_runtime_environment, - deployment_slot, - effective_slot, - elf_bytes, - account_size, - #[cfg(feature = "metrics")] - metrics, - true, /* reloading */ - ) - } - - fn new_internal( - loader_key: &Pubkey, - program_runtime_environment: ProgramRuntimeEnvironment, - deployment_slot: Slot, - effective_slot: Slot, - elf_bytes: &[u8], - account_size: usize, - #[cfg(feature = "metrics")] metrics: &mut LoadProgramMetrics, - reloading: bool, - ) -> Result> { - let entry_stats = ProgramStatistics::default(); - #[cfg(feature = "metrics")] - let load_elf_time = solana_svm_measure::measure::Measure::start("load_elf_time"); - let executable = Executable::load(elf_bytes, Arc::clone(&*program_runtime_environment))?; - - #[cfg(feature = "metrics")] - { - metrics.load_elf_us = load_elf_time.end_as_us(); - } - - if !reloading { - #[cfg(feature = "metrics")] - let verify_code_time = solana_svm_measure::measure::Measure::start("verify_code_time"); - executable.verify::()?; - #[cfg(feature = "metrics")] - { - metrics.verify_code_us = verify_code_time.end_as_us(); - } - } - - #[cfg(all(not(target_os = "windows"), target_arch = "x86_64"))] - { - let jit_compile_time = solana_svm_measure::measure::Measure::start("jit_compile_time"); - executable.jit_compile()?; - let jit_compile_time = jit_compile_time.end_as_us(); - entry_stats.jit_compiled(jit_compile_time); - #[cfg(feature = "metrics")] - { - metrics.jit_compile_us = jit_compile_time; - } - } - - Ok(Self { - deployment_slot, - account_owner: ProgramCacheEntryOwner::try_from(loader_key).unwrap(), - account_size, - effective_slot, - program: ProgramCacheEntryType::Loaded(executable), - stats: entry_stats.into(), - latest_access_slot: AtomicU64::new(0), - }) - } - - pub fn to_unloaded(&self) -> Option { - match &self.program { - ProgramCacheEntryType::Loaded(_) => {} - ProgramCacheEntryType::FailedVerification(_) - | ProgramCacheEntryType::Closed - | ProgramCacheEntryType::DelayVisibility - | ProgramCacheEntryType::Unloaded(_) - | ProgramCacheEntryType::Builtin(_) => { - return None; - } - } - Some(Self { - program: ProgramCacheEntryType::Unloaded(self.program.get_environment()?.clone()), - account_owner: self.account_owner, - account_size: self.account_size, - deployment_slot: self.deployment_slot, - effective_slot: self.effective_slot, - stats: Arc::clone(&self.stats), - latest_access_slot: AtomicU64::new(self.latest_access_slot.load(Ordering::Relaxed)), - }) - } - - /// Creates a new built-in program - pub fn new_builtin( - deployment_slot: Slot, - account_size: usize, - register_fn: BuiltinFunctionRegisterer, - ) -> Self { - let mut program = BuiltinProgram::new_builtin(); - register_fn(&mut program, "entrypoint").unwrap(); - Self { - deployment_slot, - account_owner: ProgramCacheEntryOwner::NativeLoader, - account_size, - effective_slot: deployment_slot, - program: ProgramCacheEntryType::Builtin(program), - stats: Arc::default(), - latest_access_slot: AtomicU64::new(0), - } - } - - pub fn new_tombstone( - slot: Slot, - account_owner: ProgramCacheEntryOwner, - reason: ProgramCacheEntryType, - ) -> Self { - Self::new_tombstone_with_stats(slot, account_owner, reason, Arc::default()) - } - - pub fn new_tombstone_with_stats( - slot: Slot, - account_owner: ProgramCacheEntryOwner, - reason: ProgramCacheEntryType, - stats: Arc, - ) -> Self { - let tombstone = Self { - program: reason, - account_owner, - account_size: 0, - deployment_slot: slot, - effective_slot: slot, - stats, - latest_access_slot: AtomicU64::new(0), - }; - debug_assert!(tombstone.is_tombstone()); - tombstone - } - - pub fn is_tombstone(&self) -> bool { - matches!( - self.program, - ProgramCacheEntryType::FailedVerification(_) - | ProgramCacheEntryType::Closed - | ProgramCacheEntryType::DelayVisibility - ) - } - - pub(crate) fn is_implicit_delay_visibility_tombstone(&self, slot: Slot) -> bool { - !matches!(self.program, ProgramCacheEntryType::Builtin(_)) - && self.effective_slot.saturating_sub(self.deployment_slot) - == DELAY_VISIBILITY_SLOT_OFFSET - && slot >= self.deployment_slot - && slot < self.effective_slot - } - - pub fn update_access_slot(&self, slot: Slot) { - let _ = self.latest_access_slot.fetch_max(slot, Ordering::Relaxed); - } - - /// Compute a retention score. - /// - /// Eviction uses an adapted GDSF scheme which incorporates frequency, recovery cost - /// (recompilation) and time-based decay. - /// - /// How hard should we try to retain this entry. Higher number -> retention more likely. - pub fn retention_score(&self) -> u64 { - let last_access = self.latest_access_slot.load(Ordering::Relaxed); - let recovery_cost = self.stats.compilation_time_ema.load(Ordering::Relaxed); - let frequency = self.stats.uses.load(Ordering::Relaxed); - retention_score(last_access, recovery_cost, frequency) - } - - pub fn account_owner(&self) -> Pubkey { - self.account_owner.into() - } -} - -/// See [`ProgramCacheEntry::retention_score`]. -pub(crate) const fn retention_score(last_access: u64, recovery_cost: u64, frequency: u64) -> u64 { - // Traditionally GDSF uses the following logic: - // - // on_access: - // entry.frequency += 1 - // entry.H := cache.L + (entry.cost * entry.frequency) / entry.size - // - // on_eviction: - // victim = pick_victim_minimizing_H() - // cache.L := victim.H - // - // It achieves decay by virtue of L increasing over time (and therefore the “value” of - // stored score of each entry decreasing over time.) Entry recovery and frequency, as well - // as size are otherwise also accounted for by them inflating the overall score by a bit. - // - // We adapt this algorithm slightly: we already have a kind of `L` – access slot. It does - // not include the weight of the evicted entry as the original algorithm does, that is - // *probably* fine (the author has not done any empirical experiments to verify it it - // actually matters.) - // - // Additionally we ignore the size component altogether as irrelevant and instead of - // applying entry weight linearly, we use a `log_2`. We can't use plain `weight*frequency` - // as the most heavily used entries would never ever get evicted after just some runtime, - // even if they're no longer used. With `log_2` weight and frequency can contribute to - // up-to 128 slots of "bonus" towards their retention compared to rarely used peers. - // - // Feel free to adjust the specific formulae used. - let weight = (recovery_cost as u128).wrapping_mul(frequency as u128); - let weight_log = u128::BITS.wrapping_sub(weight.leading_zeros()); - last_access.saturating_add(weight_log as u64) -} - -#[cfg(test)] -mod tests { - use { - crate::{ - loaded_programs::tests::new_test_entry_with_usage, program_metrics::ProgramStatistics, - }, - std::sync::atomic::{AtomicU64, Ordering}, - }; - - #[test] - fn test_retention_score_decay_horizon() { - let stats = ProgramStatistics { - uses: AtomicU64::new(u64::MAX), - compilation_time_ema: AtomicU64::new(u64::MAX), - ..Default::default() - }; - let program = new_test_entry_with_usage(0, 0, stats); - program.update_access_slot(1); - assert!( - dbg!(program.retention_score()) <= 129, - "retention score should remain within sensible boundaries even for very frequently \ - used entries." - ); - } - - #[test] - fn test_retention_score_frequency_preference() { - let stats = ProgramStatistics { - uses: AtomicU64::new(16), - compilation_time_ema: AtomicU64::new(1), - ..Default::default() - }; - let program = new_test_entry_with_usage(10, 11, stats); - program.update_access_slot(15); - let less_used_retention_score = program.retention_score(); - program.stats.uses.fetch_max(1024, Ordering::Relaxed); - let more_used_retention_score = program.retention_score(); - assert!( - less_used_retention_score > 15, - "frequency should count for entry retention score" - ); - assert!( - dbg!(more_used_retention_score) > dbg!(less_used_retention_score), - "retention score should prefer evicting less used entry over the more used one if \ - possible" - ); - } - - #[test] - fn test_retention_score_recovery_time_preference() { - let stats = ProgramStatistics { - uses: AtomicU64::new(1), - compilation_time_ema: AtomicU64::new(1000), - ..Default::default() - }; - let program = new_test_entry_with_usage(10, 11, stats); - program.update_access_slot(15); - let cheaper_to_compile_score = program.retention_score(); - program - .stats - .compilation_time_ema - .fetch_max(2000, Ordering::Relaxed); - let more_expensive_to_compile_score = program.retention_score(); - assert!( - cheaper_to_compile_score > 15, - "compile time should count for entry retention score" - ); - assert!( - dbg!(more_expensive_to_compile_score) > dbg!(cheaper_to_compile_score), - "retention score should prefer evicting cheaper-to-compile entries" - ); - } - - #[test] - fn test_retention_weight_metric_does_not_outweight_smaller_metric() { - // Compilation time generally stays in the scale of 4 digits, while the uses counter can - // become many millions. Neither should overshadow other too much. - let stats = ProgramStatistics { - uses: AtomicU64::new(100_000_000), - compilation_time_ema: AtomicU64::new(1000), - ..Default::default() - }; - let program = new_test_entry_with_usage(10, 11, stats); - program.update_access_slot(15); - let previous_score = program.retention_score(); - program - .stats - .compilation_time_ema - .fetch_max(2000, Ordering::Relaxed); - let new_score = program.retention_score(); - assert!( - dbg!(previous_score) != dbg!(new_score), - "retention weight components shouldn't overshadow the other due to scale differences" - ); - } -} diff --git a/solana/program-runtime/src/program_metrics.rs b/solana/program-runtime/src/program_metrics.rs deleted file mode 100644 index 31d3626..0000000 --- a/solana/program-runtime/src/program_metrics.rs +++ /dev/null @@ -1,326 +0,0 @@ -#[cfg(feature = "metrics")] -use solana_svm_timings::ExecuteDetailsTimings; -use { - crate::loaded_programs::ForkGraph, - log::{debug, log_enabled, trace}, - solana_pubkey::Pubkey, - std::{ - collections::HashMap, - sync::atomic::{AtomicU64, Ordering}, - }, -}; - -#[derive(Debug, Default)] -pub struct ProgramStatistics { - pub uses: AtomicU64, - - pub compilations: AtomicU64, - pub total_compilation_time_us: AtomicU64, - /// Exponential moving average of the compilation time. - pub compilation_time_ema: AtomicU64, - - pub jit_invocations: AtomicU64, - pub total_jit_execution_time_us: AtomicU64, - /// Exponential moving average of the JIT execution time. - pub jit_execution_time_ema: AtomicU64, - - pub interpreted_invocations: AtomicU64, - pub total_interpretation_time_us: AtomicU64, - /// Exponential moving average of the interpreted execution time. - pub interpretation_time_ema: AtomicU64, -} - -/// Number of compilation observations contributing to the the [`Self::compilation_time_ema`]. -const COMPILATION_EMA_WINDOW_SIZE: u64 = 10; -/// Number of execution observations contributing to the execution EMA stats. -const EXECUTION_EMA_WINDOW_SIZE: u64 = 500; -/// Track exponential moving average in scaled-up units. -/// -/// Doing so allows to mitigate error from rounding-towards-zero we get when using integer math. -pub(crate) const EMA_SCALE: u64 = 1_000; - -impl ProgramStatistics { - fn observe_ema(counter: &AtomicU64, duration_us: u64) { - let duration_ema = duration_us.saturating_mul(EMA_SCALE); - counter - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |ema| { - // Exponential moving average iteratively is computed as $ema' = alpha * - // observation + (1 - alpha) * ema$. This works great for floating point, but we - // want integers. For purposes of convenience we also want to really think in terms - // of simple moving average window sizes as that is easier to reason about. - // - // Exponential moving average and simple moving average of window N has a rough - // equivalence of `alpha ≈ 2 / (N + 1)`. Slotting this into our original iterative - // formula: - // - // $$ ema' = 2 / (N+1) * observation + (1 - 2/(N+1)) * ema $$ - // - // we get - // - // $$ ema' = (2*observation)/(N+1) + (N+1-2)*ema/(N+1) $$ - let (numer, denom) = const { (2, 1 + WINDOW_SIZE) }; - Some(if ema == 0 { - duration_ema - } else { - let weighted_observation = duration_ema.saturating_mul(numer); - let previous_observations = ema.saturating_mul(denom.saturating_sub(numer)); - weighted_observation - .saturating_add(previous_observations) - .checked_div(denom) - .expect("unreachable: denom is >= 1") - }) - }) - .expect("unreachable: closure always returns a Some"); - } - - /// Record information about JIT compilation. - pub fn jit_compiled(&self, duration_us: u64) { - let ord = Ordering::Relaxed; - self.compilations.fetch_add(1, ord); - self.total_compilation_time_us.fetch_add(duration_us, ord); - Self::observe_ema::(&self.compilation_time_ema, duration_us); - } - - /// Record information about JIT-compiled program having been executed. - pub fn jit_executed(&self, duration_us: u64) { - let ord = Ordering::Relaxed; - self.jit_invocations.fetch_add(1, ord); - self.total_jit_execution_time_us.fetch_add(duration_us, ord); - Self::observe_ema::(&self.jit_execution_time_ema, duration_us); - } - - /// Record information about program executed with the interpreter. - pub fn interpreter_executed(&self, duration_us: u64) { - let ord = Ordering::Relaxed; - self.interpreted_invocations.fetch_add(1, ord); - self.total_interpretation_time_us - .fetch_add(duration_us, ord); - Self::observe_ema::(&self.interpretation_time_ema, duration_us); - } - - pub fn merge_from(&self, other: &ProgramStatistics) { - let ord = Ordering::Relaxed; - self.uses.fetch_add(other.uses.load(ord), ord); - let other_compilations = other.compilations.load(ord); - let this_compilations = self.compilations.fetch_add(other_compilations, ord); - self.total_compilation_time_us - .fetch_add(other.total_compilation_time_us.load(ord), ord); - let other_jit_invocations = other.jit_invocations.load(ord); - let this_jit_invocations = self.jit_invocations.fetch_add(other_jit_invocations, ord); - self.total_jit_execution_time_us - .fetch_add(other.total_jit_execution_time_us.load(ord), ord); - let other_interpretations = other.interpreted_invocations.load(ord); - let this_interpretations = self - .interpreted_invocations - .fetch_add(other_interpretations, ord); - self.total_interpretation_time_us - .fetch_add(other.total_interpretation_time_us.load(ord), ord); - if let Some(comp_ema) = ProgramCacheStats::combined_ema::< - COMPILATION_EMA_WINDOW_SIZE, - COMPILATION_EMA_WINDOW_SIZE, - >( - &self.compilation_time_ema, - &other.compilation_time_ema, - this_compilations, - other_compilations, - ) { - self.compilation_time_ema.store(comp_ema, ord); - } - if let Some(exec_ema) = - ProgramCacheStats::combined_ema::( - &self.jit_execution_time_ema, - &other.jit_execution_time_ema, - this_jit_invocations, - other_jit_invocations, - ) - { - self.jit_execution_time_ema.store(exec_ema, ord); - } - if let Some(interp_ema) = - ProgramCacheStats::combined_ema::( - &self.interpretation_time_ema, - &other.interpretation_time_ema, - this_interpretations, - other_interpretations, - ) - { - self.interpretation_time_ema.store(interp_ema, ord); - } - } -} - -/// Global cache statistics for [ProgramCache]. -#[derive(Debug, Default)] -pub struct ProgramCacheStats { - /// a program was already in the cache - pub hits: AtomicU64, - /// a program was not found and loaded instead - pub misses: AtomicU64, - /// a compiled executable was unloaded - pub evictions: HashMap, - /// an unloaded program was loaded again (opposite of eviction) - pub reloads: AtomicU64, - /// a program was loaded or un/re/deployed - pub insertions: AtomicU64, - /// a program was loaded but can not be extracted on its own fork anymore - pub lost_insertions: AtomicU64, - /// a program which was already in the cache was reloaded by mistake - pub replacements: AtomicU64, - /// a program was only used once before being unloaded - pub one_hit_wonders: AtomicU64, - /// a program became unreachable in the fork graph because of rerooting - pub prunes_orphan: AtomicU64, - /// a program got pruned because it was not recompiled for the next epoch - pub prunes_environment: AtomicU64, - /// a program had no entries because all slot versions got pruned - pub empty_entries: AtomicU64, - /// water level of loaded entries currently cached - pub water_level: AtomicU64, -} - -impl ProgramCacheStats { - pub fn reset(&mut self) { - *self = ProgramCacheStats::default(); - } - pub fn log(&self) { - let hits = self.hits.load(Ordering::Relaxed); - let misses = self.misses.load(Ordering::Relaxed); - let evictions: u64 = self.evictions.values().sum(); - let reloads = self.reloads.load(Ordering::Relaxed); - let insertions = self.insertions.load(Ordering::Relaxed); - let lost_insertions = self.lost_insertions.load(Ordering::Relaxed); - let replacements = self.replacements.load(Ordering::Relaxed); - let one_hit_wonders = self.one_hit_wonders.load(Ordering::Relaxed); - let prunes_orphan = self.prunes_orphan.load(Ordering::Relaxed); - let prunes_environment = self.prunes_environment.load(Ordering::Relaxed); - let empty_entries = self.empty_entries.load(Ordering::Relaxed); - let water_level = self.water_level.load(Ordering::Relaxed); - debug!( - "Loaded Programs Cache Stats -- Hits: {hits}, Misses: {misses}, Evictions: \ - {evictions}, Reloads: {reloads}, Insertions: {insertions}, Lost-Insertions: \ - {lost_insertions}, Replacements: {replacements}, One-Hit-Wonders: {one_hit_wonders}, \ - Prunes-Orphan: {prunes_orphan}, Prunes-Environment: {prunes_environment}, Empty: \ - {empty_entries}, Water-Level: {water_level}" - ); - - if log_enabled!(log::Level::Trace) && !self.evictions.is_empty() { - let mut evictions = self.evictions.iter().collect::>(); - evictions.sort_by_key(|e| e.1); - let evictions = evictions - .into_iter() - .rev() - .map(|(program_id, evictions)| { - format!(" {:<44} {}", program_id.to_string(), evictions) - }) - .collect::>(); - let evictions = evictions.join("\n"); - trace!( - "Eviction Details:\n {:<44} {}\n{}", - "Program", "Count", evictions - ); - } - } - - fn combined_ema( - into_ema: &AtomicU64, - from_ema: &AtomicU64, - into_observations: u64, - from_observations: u64, - ) -> Option { - // This is a mild non-sense, but there is no good mathematically rigorous way to merge - // two independent EMA trackers AFAICT and this is the best I (nagisa) could come up - // with… - let other_ema_val = from_ema.load(Ordering::Relaxed); - let other_ema_weight = std::cmp::max(WINDOW1, from_observations); - let this_ema_val = into_ema.load(Ordering::Relaxed); - let this_ema_weight = std::cmp::max(WINDOW2, into_observations); - other_ema_val - .wrapping_mul(other_ema_weight) - .wrapping_add(this_ema_val.wrapping_mul(this_ema_weight)) - .checked_div(other_ema_weight.wrapping_add(this_ema_weight)) - } -} - -#[cfg(feature = "metrics")] -/// Time measurements for loading a single [ProgramCacheEntry]. -#[derive(Debug, Default)] -pub struct LoadProgramMetrics { - /// Program address, but as text - pub program_id: String, - /// Microseconds it took to `create_program_runtime_environment` - pub register_syscalls_us: u64, - /// Microseconds it took to `Executable::::load` - pub load_elf_us: u64, - /// Microseconds it took to `executable.verify::` - pub verify_code_us: u64, - /// Microseconds it took to `executable.jit_compile` - pub jit_compile_us: u64, -} - -#[cfg(feature = "metrics")] -impl LoadProgramMetrics { - pub fn submit_datapoint(&self, timings: &mut ExecuteDetailsTimings) { - timings.create_executor_register_syscalls_us += self.register_syscalls_us; - timings.create_executor_load_elf_us += self.load_elf_us; - timings.create_executor_verify_code_us += self.verify_code_us; - timings.create_executor_jit_compile_us += self.jit_compile_us; - } -} - -impl crate::loaded_programs::ProgramCache { - /// Log per-entry statistics for each entry in the global cache. - #[cfg(feature = "dev-context-only-utils")] - pub fn output_entry_stats(&self) { - use {crate::program_cache_entry::ProgramCacheEntryType, std::fmt::Write}; - // The entry stats can become very verbose after some runtime. Rather than dumping them - // to the log, we'd rather maintain a continuously updated file instead... - static ENTRY_STAT_PATH: std::sync::LazyLock> = - std::sync::LazyLock::new(|| std::env::var_os("AGAVE_PROGRAM_CACHE_ENTRY_STATS_PATH")); - let Some(stat_path) = &*ENTRY_STAT_PATH else { - log::trace!("Set AGAVE_PROGRAM_CACHE_ENTRY_STATS_PATH to write per-entry stats"); - return; - }; - let mut output = String::new(); - let entries = self.get_flattened_entries_for_tests(); - for (addr, entry) in entries { - let entry_ty = match &entry.program { - ProgramCacheEntryType::FailedVerification(_) => "FailedVerification", - ProgramCacheEntryType::Closed => "Closed", - ProgramCacheEntryType::DelayVisibility => "DelayVisibility", - ProgramCacheEntryType::Unloaded(_) => "Unloaded", - ProgramCacheEntryType::Builtin(_) => "Builtin", - #[cfg(not(all(not(target_os = "windows"), target_arch = "x86_64")))] - ProgramCacheEntryType::Loaded(_) => "Loaded", - #[cfg(all(not(target_os = "windows"), target_arch = "x86_64"))] - ProgramCacheEntryType::Loaded(executable) => { - if executable.get_compiled_program().is_some() { - "JitCompiled" - } else { - "Loaded" - } - } - }; - let stats = &entry.stats; - let uses = stats.uses.load(Ordering::Relaxed); - let compiles = stats.compilations.load(Ordering::Relaxed); - let comptime = stats.total_compilation_time_us.load(Ordering::Relaxed); - let comptime_ema = stats.compilation_time_ema.load(Ordering::Relaxed) / EMA_SCALE; - let invokes = stats.jit_invocations.load(Ordering::Relaxed); - let jittime = stats.total_jit_execution_time_us.load(Ordering::Relaxed); - let jittime_ema = stats.jit_execution_time_ema.load(Ordering::Relaxed) / EMA_SCALE; - let interps = stats.interpreted_invocations.load(Ordering::Relaxed); - let interptime = stats.total_interpretation_time_us.load(Ordering::Relaxed); - let interpema = stats.interpretation_time_ema.load(Ordering::Relaxed) / EMA_SCALE; - let _ = writeln!( - &mut output, - "{addr},{entry_ty},{uses},{compiles},{comptime},{comptime_ema},{invokes},\ - {jittime},{jittime_ema},{interps},{interptime},{interpema}" - ); - } - if let Err(e) = std::fs::write(stat_path, output) { - log::info!("Writing entry stats to {stat_path:?} failed: {e:?}"); - } else { - log::debug!("Entry stats written to {stat_path:?}"); - } - } -} diff --git a/solana/program-runtime/src/serialization.rs b/solana/program-runtime/src/serialization.rs index 0ddd77a..b9ef3ef 100644 --- a/solana/program-runtime/src/serialization.rs +++ b/solana/program-runtime/src/serialization.rs @@ -1,7 +1,7 @@ #![allow(clippy::arithmetic_side_effects)] use { - crate::memory_context::SerializedAccountMetadata, + crate::invoke_context::SerializedAccountMetadata, solana_instruction::error::InstructionError, solana_program_entrypoint::{BPF_ALIGN_OF_U128, MAX_PERMITTED_DATA_INCREASE, NON_DUP_MARKER}, solana_pubkey::Pubkey, @@ -19,7 +19,7 @@ use { std::mem::{self, size_of}, }; -/// Modifies the memory mapping in serialization and CPI return for virtual_address_space_adjustments +/// Modifies an existing memory mapping region to point at account data. pub fn modify_memory_region_of_account( account: &mut BorrowedInstructionAccount<'_, '_>, region: &mut MemoryRegion, @@ -34,7 +34,7 @@ pub fn modify_memory_region_of_account( } } -/// Creates the memory mapping in serialization and CPI return for account_data_direct_mapping +/// Creates the memory mapping in serialization and CPI return for direct account data. pub fn create_memory_region_of_account( account: &mut BorrowedInstructionAccount<'_, '_>, vaddr: u64, @@ -51,7 +51,7 @@ pub fn create_memory_region_of_account( Ok(memory_region) } -#[expect(dead_code)] +#[allow(dead_code)] enum SerializeAccount<'a, 'ix_data> { Account(IndexOfAccount, BorrowedInstructionAccount<'a, 'ix_data>), Duplicate(IndexOfAccount), @@ -63,26 +63,16 @@ struct Serializer { vaddr: u64, region_start: usize, is_loader_v1: bool, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, } impl Serializer { - fn new( - size: usize, - start_addr: u64, - is_loader_v1: bool, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, - ) -> Serializer { + fn new(size: usize, start_addr: u64, is_loader_v1: bool) -> Serializer { Serializer { buffer: AlignedMemory::with_capacity(size), regions: Vec::new(), region_start: 0, vaddr: start_addr, is_loader_v1, - virtual_address_space_adjustments, - account_data_direct_mapping, } } @@ -129,71 +119,38 @@ impl Serializer { &mut self, account: &mut BorrowedInstructionAccount<'_, '_>, ) -> Result { - if !self.virtual_address_space_adjustments { - let vm_data_addr = self.vaddr.saturating_add(self.buffer.len() as u64); - self.write_all(account.get_data()); - if !self.is_loader_v1 { - let align_offset = - (account.get_data().len() as *const u8).align_offset(BPF_ALIGN_OF_U128); - self.fill_write(MAX_PERMITTED_DATA_INCREASE + align_offset, 0) - .map_err(|_| InstructionError::InvalidArgument)?; - } - Ok(vm_data_addr) + self.push_region(); + let vm_data_addr = self.vaddr; + let address_space_reserved_for_account = if !self.is_loader_v1 { + account.get_data().len().saturating_add(MAX_PERMITTED_DATA_INCREASE) } else { - self.push_region(); - let vm_data_addr = self.vaddr; - if !self.account_data_direct_mapping { - self.write_all(account.get_data()); - if !self.is_loader_v1 { - self.fill_write(MAX_PERMITTED_DATA_INCREASE, 0) - .map_err(|_| InstructionError::InvalidArgument)?; - } - } - let address_space_reserved_for_account = if !self.is_loader_v1 { - account - .get_data() - .len() - .saturating_add(MAX_PERMITTED_DATA_INCREASE) - } else { - account.get_data().len() - }; - if address_space_reserved_for_account > 0 { - if !self.account_data_direct_mapping { - self.push_region(); - let region = self.regions.last_mut().unwrap(); - modify_memory_region_of_account(account, region); - } else { - let new_region = create_memory_region_of_account(account, self.vaddr)?; - self.vaddr += address_space_reserved_for_account as u64; - self.regions.push(new_region); - } - } - if !self.is_loader_v1 { - let align_offset = - (account.get_data().len() as *const u8).align_offset(BPF_ALIGN_OF_U128); - if !self.account_data_direct_mapping { - self.fill_write(align_offset, 0) - .map_err(|_| InstructionError::InvalidArgument)?; - } else { - // The deserialization code is going to align the vm_addr to - // BPF_ALIGN_OF_U128. Always add one BPF_ALIGN_OF_U128 worth of - // padding and shift the start of the next region, so that once - // vm_addr is aligned, the corresponding host_addr is aligned - // too. - self.fill_write(BPF_ALIGN_OF_U128, 0) - .map_err(|_| InstructionError::InvalidArgument)?; - self.region_start += BPF_ALIGN_OF_U128.saturating_sub(align_offset); - } - } - Ok(vm_data_addr) + account.get_data().len() + }; + if address_space_reserved_for_account > 0 { + let new_region = create_memory_region_of_account(account, self.vaddr)?; + self.vaddr += address_space_reserved_for_account as u64; + self.regions.push(new_region); + } + if !self.is_loader_v1 { + let align_offset = + (account.get_data().len() as *const u8).align_offset(BPF_ALIGN_OF_U128); + // The deserialization code is going to align the vm_addr to + // BPF_ALIGN_OF_U128. Always add one BPF_ALIGN_OF_U128 worth of + // padding and shift the start of the next region, so that once + // vm_addr is aligned, the corresponding host_addr is aligned too. + self.fill_write(BPF_ALIGN_OF_U128, 0) + .map_err(|_| InstructionError::InvalidArgument)?; + self.region_start += BPF_ALIGN_OF_U128.saturating_sub(align_offset); } + Ok(vm_data_addr) } fn push_region(&mut self) { let range = self.region_start..self.buffer.len(); - let region_slice = self.buffer.as_slice_mut().get_mut(range.clone()).unwrap(); - self.regions - .push(MemoryRegion::new(&raw mut region_slice[..], self.vaddr)); + self.regions.push(MemoryRegion::new( + &raw mut self.buffer.as_slice_mut().get_mut(range.clone()).unwrap()[..], + self.vaddr, + )); self.region_start = range.end; self.vaddr += range.len() as u64; } @@ -207,12 +164,7 @@ impl Serializer { fn debug_assert_alignment(&self) { debug_assert!( self.is_loader_v1 - || self - .buffer - .as_slice() - .as_ptr_range() - .end - .align_offset(mem::align_of::()) + || self.buffer.as_slice().as_ptr_range().end.align_offset(mem::align_of::()) == 0 ); } @@ -220,8 +172,6 @@ impl Serializer { pub fn serialize_parameters( instruction_context: &InstructionContext, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, direct_account_pointers_in_program_input: bool, ) -> Result< ( @@ -267,8 +217,6 @@ pub fn serialize_parameters( accounts, instruction_context.get_instruction_data(), &program_id, - virtual_address_space_adjustments, - account_data_direct_mapping, ) } else { // Used by loader-v2 (bpf_loader) and loader-v3 (bpf_loader_upgradeable) @@ -276,8 +224,6 @@ pub fn serialize_parameters( accounts, instruction_context.get_instruction_data(), &program_id, - virtual_address_space_adjustments, - account_data_direct_mapping, // SIMD-0449: only available on ABIv1 direct_account_pointers_in_program_input, ) @@ -286,8 +232,6 @@ pub fn serialize_parameters( pub fn deserialize_parameters( instruction_context: &InstructionContext, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, buffer: &[u8], accounts_metadata: &[SerializedAccountMetadata], ) -> Result<(), InstructionError> { @@ -296,22 +240,10 @@ pub fn deserialize_parameters( let account_lengths = accounts_metadata.iter().map(|a| a.original_data_len); if is_loader_deprecated { // Used by loader-v1 (bpf_loader_deprecated) - deserialize_parameters_for_abiv0( - instruction_context, - virtual_address_space_adjustments, - account_data_direct_mapping, - buffer, - account_lengths, - ) + deserialize_parameters_for_abiv0(instruction_context, buffer, account_lengths) } else { // Used by loader-v2 (bpf_loader) and loader-v3 (bpf_loader_upgradeable) - deserialize_parameters_for_abiv1( - instruction_context, - virtual_address_space_adjustments, - account_data_direct_mapping, - buffer, - account_lengths, - ) + deserialize_parameters_for_abiv1(instruction_context, buffer, account_lengths) } } @@ -319,8 +251,6 @@ fn serialize_parameters_for_abiv0( accounts: Vec, instruction_data: &[u8], program_id: &Pubkey, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, ) -> Result< ( AlignedMemory, @@ -336,7 +266,7 @@ fn serialize_parameters_for_abiv0( size += 1; // dup match account { SerializeAccount::Duplicate(_) => {} - SerializeAccount::Account(_, account) => { + SerializeAccount::Account(_, _) => { size += size_of::() // is_signer + size_of::() // is_writable + size_of::() // key @@ -345,9 +275,6 @@ fn serialize_parameters_for_abiv0( + size_of::() // owner + size_of::() // executable + size_of::(); // rent_epoch - if !(virtual_address_space_adjustments && account_data_direct_mapping) { - size += account.get_data().len(); - } } } } @@ -355,13 +282,7 @@ fn serialize_parameters_for_abiv0( + instruction_data.len() // instruction data + size_of::(); // program id - let mut s = Serializer::new( - size, - MM_INPUT_START, - true, - virtual_address_space_adjustments, - account_data_direct_mapping, - ); + let mut s = Serializer::new(size, MM_INPUT_START, true); let mut accounts_metadata: Vec = Vec::with_capacity(accounts.len()); s.write::((accounts.len() as u64).to_le()); @@ -380,7 +301,7 @@ fn serialize_parameters_for_abiv0( s.write::((account.get_data().len() as u64).to_le()); let vm_data_addr = s.write_account(&mut account)?; let vm_owner_addr = s.write_all(account.get_owner().as_ref()); - #[expect(deprecated)] + #[allow(deprecated)] s.write::(account.is_executable() as u8); let rent_epoch = u64::MAX; s.write::(rent_epoch.to_le()); @@ -410,8 +331,6 @@ fn serialize_parameters_for_abiv0( fn deserialize_parameters_for_abiv0>( instruction_context: &InstructionContext, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, buffer: &[u8], account_lengths: I, ) -> Result<(), InstructionError> { @@ -439,28 +358,9 @@ fn deserialize_parameters_for_abiv0>( } start += size_of::() // lamports + size_of::(); // data length - if !virtual_address_space_adjustments { - let data = buffer - .get(start..start + pre_len) - .ok_or(InstructionError::InvalidArgument)?; - // The redundant check helps to avoid the expensive data comparison if we can - match borrowed_account.can_data_be_resized(pre_len) { - Ok(()) => borrowed_account.set_data_from_slice(data)?, - Err(err) if borrowed_account.get_data() != data => return Err(err), - _ => {} - } - } else if !account_data_direct_mapping && borrowed_account.can_data_be_changed().is_ok() - { - let data = buffer - .get(start..start + pre_len) - .ok_or(InstructionError::InvalidArgument)?; - borrowed_account.set_data_from_slice(data)?; - } else if borrowed_account.get_data().len() != pre_len { + if borrowed_account.get_data().len() != pre_len { borrowed_account.set_data_length(pre_len)?; } - if !(virtual_address_space_adjustments && account_data_direct_mapping) { - start += pre_len; // data - } start += size_of::() // owner + size_of::() // executable + size_of::(); // rent_epoch @@ -473,8 +373,6 @@ fn serialize_parameters_for_abiv1( accounts: Vec, instruction_data: &[u8], program_id: &Pubkey, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, direct_account_pointers_program_input: bool, ) -> Result< ( @@ -492,8 +390,7 @@ fn serialize_parameters_for_abiv1( size += 1; // dup match account { SerializeAccount::Duplicate(_) => size += 7, // padding to 64-bit aligned - SerializeAccount::Account(_, account) => { - let data_len = account.get_data().len(); + SerializeAccount::Account(_, _) => { size += size_of::() // is_signer + size_of::() // is_writable + size_of::() // executable @@ -503,13 +400,7 @@ fn serialize_parameters_for_abiv1( + size_of::() // lamports + size_of::() // data len + size_of::(); // rent epoch - if !(virtual_address_space_adjustments && account_data_direct_mapping) { - size += data_len - + MAX_PERMITTED_DATA_INCREASE - + (data_len as *const u8).align_offset(BPF_ALIGN_OF_U128); - } else { - size += BPF_ALIGN_OF_U128; - } + size += BPF_ALIGN_OF_U128; } } } @@ -526,13 +417,7 @@ fn serialize_parameters_for_abiv1( None }; - let mut s = Serializer::new( - size, - MM_INPUT_START, - false, - virtual_address_space_adjustments, - account_data_direct_mapping, - ); + let mut s = Serializer::new(size, MM_INPUT_START, false); // Serialize into the buffer s.write::((accounts.len() as u64).to_le()); @@ -542,7 +427,7 @@ fn serialize_parameters_for_abiv1( let vm_addr = s.write::(NON_DUP_MARKER); s.write::(borrowed_account.is_signer() as u8); s.write::(borrowed_account.is_writable() as u8); - #[expect(deprecated)] + #[allow(deprecated)] s.write::(borrowed_account.is_executable() as u8); s.write_all(&[0u8, 0, 0, 0]); let vm_key_addr = s.write_all(borrowed_account.get_key().as_ref()); @@ -575,8 +460,7 @@ fn serialize_parameters_for_abiv1( if let Some(offset) = account_pointers_offset { // Add padding before the account pointer array to reach 8-byte alignment // (BPF_ALIGN_OF_U128). - s.fill_write(offset, 0) - .map_err(|_| InstructionError::InvalidArgument)?; + s.fill_write(offset, 0).map_err(|_| InstructionError::InvalidArgument)?; for entry in accounts_metadata.iter() { s.write::(entry.vm_addr.to_le()); } @@ -593,8 +477,6 @@ fn serialize_parameters_for_abiv1( fn deserialize_parameters_for_abiv1>( instruction_context: &InstructionContext, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, buffer: &[u8], account_lengths: I, ) -> Result<(), InstructionError> { @@ -641,34 +523,11 @@ fn deserialize_parameters_for_abiv1>( { return Err(InstructionError::InvalidRealloc); } - if !virtual_address_space_adjustments { - let data = buffer - .get(start..start + post_len) - .ok_or(InstructionError::InvalidArgument)?; - // The redundant check helps to avoid the expensive data comparison if we can - match borrowed_account.can_data_be_resized(post_len) { - Ok(()) => borrowed_account.set_data_from_slice(data)?, - Err(err) if borrowed_account.get_data() != data => return Err(err), - _ => {} - } - } else if !account_data_direct_mapping && borrowed_account.can_data_be_changed().is_ok() - { - let data = buffer - .get(start..start + post_len) - .ok_or(InstructionError::InvalidArgument)?; - borrowed_account.set_data_from_slice(data)?; - } else if borrowed_account.get_data().len() != post_len { + if borrowed_account.get_data().len() != post_len { borrowed_account.set_data_length(post_len)?; } - start += if !(virtual_address_space_adjustments && account_data_direct_mapping) { - let alignment_offset = (pre_len as *const u8).align_offset(BPF_ALIGN_OF_U128); - pre_len // data - .saturating_add(MAX_PERMITTED_DATA_INCREASE) // realloc padding - .saturating_add(alignment_offset) - } else { - // See Serializer::write_account() as to why we have this - BPF_ALIGN_OF_U128 - }; + // See Serializer::write_account() as to why we have this padding. + start += BPF_ALIGN_OF_U128; start += size_of::(); // rent_epoch if borrowed_account.get_owner().to_bytes() != owner { // Change the owner at the end so that we are allowed to change the lamports and data before @@ -685,7 +544,10 @@ mod tests { use { super::*, crate::with_mock_invoke_context, - solana_account::{Account, AccountSharedData, ReadableAccount}, + solana_account::{ + Account, AccountSharedData, CoWAccount, ReadableAccount, + testkit::{active_borrowed_data, borrowed_account_buffer, borrowed_shared_data}, + }, solana_account_info::AccountInfo, solana_program_entrypoint::deserialize, solana_rent::Rent, @@ -735,295 +597,116 @@ mod tests { name: &'static str, } - for virtual_address_space_adjustments in [false, true] { - for TestCase { - num_ix_accounts, - append_dup_account, - expected_err, - name, - } in [ - TestCase { - name: "serialize max accounts with cap", - num_ix_accounts: MAX_ACCOUNTS_PER_INSTRUCTION, - append_dup_account: false, - expected_err: None, - }, - TestCase { - name: "serialize too many accounts with cap", - num_ix_accounts: MAX_ACCOUNTS_PER_INSTRUCTION + 1, - append_dup_account: false, - expected_err: Some(InstructionError::MaxAccountsExceeded), - }, - TestCase { - name: "serialize too many accounts and append dup with cap", - num_ix_accounts: MAX_ACCOUNTS_PER_INSTRUCTION, - append_dup_account: true, - expected_err: Some(InstructionError::MaxAccountsExceeded), - }, - ] { - let program_id = solana_pubkey::new_rand(); - let mut transaction_accounts = vec![( - program_id, + for TestCase { + num_ix_accounts, + append_dup_account, + expected_err, + name, + } in [ + TestCase { + name: "serialize max accounts with cap", + num_ix_accounts: MAX_ACCOUNTS_PER_INSTRUCTION, + append_dup_account: false, + expected_err: None, + }, + TestCase { + name: "serialize too many accounts with cap", + num_ix_accounts: MAX_ACCOUNTS_PER_INSTRUCTION + 1, + append_dup_account: false, + expected_err: Some(InstructionError::MaxAccountsExceeded), + }, + TestCase { + name: "serialize too many accounts and append dup with cap", + num_ix_accounts: MAX_ACCOUNTS_PER_INSTRUCTION, + append_dup_account: true, + expected_err: Some(InstructionError::MaxAccountsExceeded), + }, + ] { + let program_id = solana_pubkey::new_rand(); + let mut transaction_accounts = vec![( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + )]; + for _ in 0..num_ix_accounts { + transaction_accounts.push(( + Pubkey::new_unique(), AccountSharedData::from(Account { lamports: 0, data: vec![], - owner: bpf_loader::id(), - executable: true, + owner: program_id, + executable: false, rent_epoch: 0, }), - )]; - for _ in 0..num_ix_accounts { - transaction_accounts.push(( - Pubkey::new_unique(), - AccountSharedData::from(Account { - lamports: 0, - data: vec![], - owner: program_id, - executable: false, - rent_epoch: 0, - }), - )); - } + )); + } - let transaction_accounts_indexes: Vec = - (0..num_ix_accounts as u16).collect(); - let mut instruction_accounts = - deduplicated_instruction_accounts(&transaction_accounts_indexes, |_| false); - if append_dup_account { - instruction_accounts.push(instruction_accounts.last().cloned().unwrap()); - } - let instruction_data = vec![]; - - with_mock_invoke_context!( - invoke_context, - transaction_context, - transaction_accounts - ); - if instruction_accounts.len() > MAX_ACCOUNTS_PER_INSTRUCTION { - // Special case implementation of configure_next_instruction_for_tests() - // which avoids the overflow when constructing the dedup_map - // by simply not filling it. - let dedup_map = vec![u16::MAX; MAX_ACCOUNTS_PER_TRANSACTION]; - invoke_context - .transaction_context - .configure_instruction_at_index( - 0, - 0, - instruction_accounts, - dedup_map, - Cow::Owned(instruction_data.clone()), - Some(0), - ) - .unwrap(); - } else { - invoke_context - .transaction_context - .configure_top_level_instruction_for_tests( - 0, - instruction_accounts, - instruction_data.clone(), - ) - .unwrap(); - } - invoke_context.push().unwrap(); - let instruction_context = invoke_context + let transaction_accounts_indexes: Vec = + (0..num_ix_accounts as u16).collect(); + let mut instruction_accounts = + deduplicated_instruction_accounts(&transaction_accounts_indexes, |_| false); + if append_dup_account { + instruction_accounts.push(instruction_accounts.last().cloned().unwrap()); + } + let instruction_data = vec![]; + + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + if instruction_accounts.len() > MAX_ACCOUNTS_PER_INSTRUCTION { + // Special case implementation of configure_next_instruction_for_tests() + // which avoids the overflow when constructing the dedup_map + // by simply not filling it. + let dedup_map = vec![u16::MAX; MAX_ACCOUNTS_PER_TRANSACTION]; + invoke_context .transaction_context - .get_current_instruction_context() + .configure_instruction_at_index( + 0, + 0, + instruction_accounts, + dedup_map, + Cow::Owned(instruction_data.clone()), + Some(0), + ) .unwrap(); - - let serialization_result = serialize_parameters( - &instruction_context, - virtual_address_space_adjustments, - false, // account_data_direct_mapping - direct_account_pointers_in_program_input, - ); - assert_eq!( - serialization_result.as_ref().err(), - expected_err.as_ref(), - "{name} test case failed", - ); - if expected_err.is_some() { - continue; - } - - let (mut serialized, regions, _account_lengths, _instruction_data_offset) = - serialization_result.unwrap(); - let mut serialized_regions = concat_regions(®ions); - let (de_program_id, de_accounts, de_instruction_data) = unsafe { - deserialize( - if !virtual_address_space_adjustments { - serialized.as_slice_mut() - } else { - serialized_regions.as_slice_mut() - } - .first_mut() - .unwrap() as *mut u8, + } else { + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 0, + instruction_accounts, + instruction_data.clone(), ) - }; - assert_eq!(de_program_id, &program_id); - assert_eq!(de_instruction_data, &instruction_data); - for account_info in de_accounts { - let index_in_transaction = invoke_context - .transaction_context - .find_index_of_account(account_info.key) - .unwrap(); - let account = invoke_context - .transaction_context - .accounts() - .try_borrow(index_in_transaction) - .unwrap(); - assert_eq!(account.lamports(), account_info.lamports()); - assert_eq!(account.data(), &account_info.data.borrow()[..]); - assert_eq!(account.owner(), account_info.owner); - assert_eq!(account.executable(), account_info.executable); - #[allow(deprecated)] - { - // Using the sdk entrypoint, the rent-epoch is skipped - assert_eq!(0, account_info._unused); - } - } + .unwrap(); } - } - } - - #[test_case(false; "direct_account_pointers_in_program_input disabled")] - #[test_case(true; "direct_account_pointers_in_program_input enabled")] - fn test_serialize_parameters(direct_account_pointers_in_program_input: bool) { - for virtual_address_space_adjustments in [false, true] { - let program_id = solana_pubkey::new_rand(); - let transaction_accounts = vec![ - ( - program_id, - AccountSharedData::from(Account { - lamports: 0, - data: vec![], - owner: bpf_loader::id(), - executable: true, - rent_epoch: 0, - }), - ), - ( - solana_pubkey::new_rand(), - AccountSharedData::from(Account { - lamports: 1, - data: vec![1u8, 2, 3, 4, 5], - owner: bpf_loader::id(), - executable: false, - rent_epoch: 100, - }), - ), - ( - solana_pubkey::new_rand(), - AccountSharedData::from(Account { - lamports: 2, - data: vec![11u8, 12, 13, 14, 15, 16, 17, 18, 19], - owner: bpf_loader::id(), - executable: true, - rent_epoch: 200, - }), - ), - ( - solana_pubkey::new_rand(), - AccountSharedData::from(Account { - lamports: 3, - data: vec![], - owner: bpf_loader::id(), - executable: false, - rent_epoch: 3100, - }), - ), - ( - solana_pubkey::new_rand(), - AccountSharedData::from(Account { - lamports: 4, - data: vec![1u8, 2, 3, 4, 5], - owner: bpf_loader::id(), - executable: false, - rent_epoch: 100, - }), - ), - ( - solana_pubkey::new_rand(), - AccountSharedData::from(Account { - lamports: 5, - data: vec![11u8, 12, 13, 14, 15, 16, 17, 18, 19], - owner: bpf_loader::id(), - executable: true, - rent_epoch: 200, - }), - ), - ( - solana_pubkey::new_rand(), - AccountSharedData::from(Account { - lamports: 6, - data: vec![], - owner: bpf_loader::id(), - executable: false, - rent_epoch: 3100, - }), - ), - ( - program_id, - AccountSharedData::from(Account { - lamports: 0, - data: vec![], - owner: bpf_loader_deprecated::id(), - executable: true, - rent_epoch: 0, - }), - ), - ]; - let instruction_accounts = - deduplicated_instruction_accounts(&[1, 1, 2, 3, 4, 4, 5, 6], |index| index >= 4); - let instruction_data = vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; - let original_accounts = transaction_accounts.clone(); - with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); - invoke_context - .transaction_context - .configure_top_level_instruction_for_tests( - 0, - instruction_accounts.clone(), - instruction_data.clone(), - ) - .unwrap(); invoke_context.push().unwrap(); - let instruction_context = invoke_context - .transaction_context - .get_current_instruction_context() - .unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); - // check serialize_parameters_for_abiv1 - let (mut serialized, regions, accounts_metadata, _instruction_data_offset) = - serialize_parameters( - &instruction_context, - virtual_address_space_adjustments, - false, // account_data_direct_mapping - direct_account_pointers_in_program_input, - ) - .unwrap(); + let serialization_result = serialize_parameters( + &instruction_context, + direct_account_pointers_in_program_input, + ); + assert_eq!( + serialization_result.as_ref().err(), + expected_err.as_ref(), + "{name} test case failed", + ); + if expected_err.is_some() { + continue; + } + let (_serialized, regions, _account_lengths, _instruction_data_offset) = + serialization_result.unwrap(); let mut serialized_regions = concat_regions(®ions); - if !virtual_address_space_adjustments { - assert_eq!(serialized.as_slice(), serialized_regions.as_slice()); - } let (de_program_id, de_accounts, de_instruction_data) = unsafe { - deserialize( - if !virtual_address_space_adjustments { - serialized.as_slice_mut() - } else { - serialized_regions.as_slice_mut() - } - .first_mut() - .unwrap() as *mut u8, - ) + deserialize(serialized_regions.as_slice_mut().first_mut().unwrap() as *mut u8) }; - - assert_eq!(&program_id, de_program_id); - assert_eq!(instruction_data, de_instruction_data); - assert_eq!( - (de_instruction_data.first().unwrap() as *const u8).align_offset(BPF_ALIGN_OF_U128), - 0 - ); + assert_eq!(de_program_id, &program_id); + assert_eq!(de_instruction_data, &instruction_data); for account_info in de_accounts { let index_in_transaction = invoke_context .transaction_context @@ -1043,118 +726,240 @@ mod tests { // Using the sdk entrypoint, the rent-epoch is skipped assert_eq!(0, account_info._unused); } - - assert_eq!( - (*account_info.lamports.borrow() as *const u64).align_offset(BPF_ALIGN_OF_U128), - 0 - ); - assert_eq!( - account_info - .data - .borrow() - .as_ptr() - .align_offset(BPF_ALIGN_OF_U128), - 0 - ); } + } + } - deserialize_parameters( + #[test_case(false; "direct_account_pointers_in_program_input disabled")] + #[test_case(true; "direct_account_pointers_in_program_input enabled")] + fn test_serialize_parameters(direct_account_pointers_in_program_input: bool) { + let program_id = solana_pubkey::new_rand(); + let transaction_accounts = vec![ + ( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 1, + data: vec![1u8, 2, 3, 4, 5], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 100, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 2, + data: vec![11u8, 12, 13, 14, 15, 16, 17, 18, 19], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 200, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 3, + data: vec![], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 3100, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 4, + data: vec![1u8, 2, 3, 4, 5], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 100, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 5, + data: vec![11u8, 12, 13, 14, 15, 16, 17, 18, 19], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 200, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 6, + data: vec![], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 3100, + }), + ), + ( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader_deprecated::id(), + executable: true, + rent_epoch: 0, + }), + ), + ]; + let instruction_accounts = + deduplicated_instruction_accounts(&[1, 1, 2, 3, 4, 4, 5, 6], |index| index >= 4); + let instruction_data = vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; + let original_accounts = transaction_accounts.clone(); + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 0, + instruction_accounts.clone(), + instruction_data.clone(), + ) + .unwrap(); + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + + // check serialize_parameters_for_abiv1 + let (serialized, regions, accounts_metadata, _instruction_data_offset) = + serialize_parameters( &instruction_context, - virtual_address_space_adjustments, - false, // account_data_direct_mapping - serialized.as_slice(), - &accounts_metadata, + direct_account_pointers_in_program_input, ) .unwrap(); - for (index_in_transaction, (_key, original_account)) in - original_accounts.iter().enumerate() - { - let account = invoke_context - .transaction_context - .accounts() - .try_borrow(index_in_transaction as IndexOfAccount) - .unwrap(); - assert_eq!(&*account, original_account); - } - invoke_context.pop().unwrap(); - // check serialize_parameters_for_abiv0 - invoke_context + let mut serialized_regions = concat_regions(®ions); + let (de_program_id, de_accounts, de_instruction_data) = unsafe { + deserialize(serialized_regions.as_slice_mut().first_mut().unwrap() as *mut u8) + }; + + assert_eq!(&program_id, de_program_id); + assert_eq!(instruction_data, de_instruction_data); + assert_eq!( + (de_instruction_data.first().unwrap() as *const u8).align_offset(BPF_ALIGN_OF_U128), + 0 + ); + for account_info in de_accounts { + let index_in_transaction = invoke_context .transaction_context - .configure_top_level_instruction_for_tests( - 7, - instruction_accounts, - instruction_data.clone(), - ) + .find_index_of_account(account_info.key) .unwrap(); - invoke_context.push().unwrap(); - let instruction_context = invoke_context + let account = invoke_context .transaction_context - .get_current_instruction_context() + .accounts() + .try_borrow(index_in_transaction) .unwrap(); + assert_eq!(account.lamports(), account_info.lamports()); + assert_eq!(account.data(), &account_info.data.borrow()[..]); + assert_eq!(account.owner(), account_info.owner); + assert_eq!(account.executable(), account_info.executable); + #[allow(deprecated)] + { + // Using the sdk entrypoint, the rent-epoch is skipped + assert_eq!(0, account_info._unused); + } - let (mut serialized, regions, account_lengths, _instruction_data_offset) = - serialize_parameters( - &instruction_context, - virtual_address_space_adjustments, - false, // account_data_direct_mapping - direct_account_pointers_in_program_input, - ) + assert_eq!( + (*account_info.lamports.borrow() as *const u64).align_offset(BPF_ALIGN_OF_U128), + 0 + ); + assert_eq!( + account_info.data.borrow().as_ptr().align_offset(BPF_ALIGN_OF_U128), + 0 + ); + } + + deserialize_parameters( + &instruction_context, + serialized.as_slice(), + &accounts_metadata, + ) + .unwrap(); + for (index_in_transaction, (_key, original_account)) in original_accounts.iter().enumerate() + { + let account = invoke_context + .transaction_context + .accounts() + .try_borrow(index_in_transaction as IndexOfAccount) .unwrap(); - let mut serialized_regions = concat_regions(®ions); + assert_eq!(&*account, original_account); + } - let (de_program_id, de_accounts, de_instruction_data) = unsafe { - deserialize_for_abiv0( - if !virtual_address_space_adjustments { - serialized.as_slice_mut() - } else { - serialized_regions.as_slice_mut() - } - .first_mut() - .unwrap() as *mut u8, - ) - }; - assert_eq!(&program_id, de_program_id); - assert_eq!(instruction_data, de_instruction_data); - for account_info in de_accounts { - let index_in_transaction = invoke_context - .transaction_context - .find_index_of_account(account_info.key) - .unwrap(); - let account = invoke_context - .transaction_context - .accounts() - .try_borrow(index_in_transaction) - .unwrap(); - assert_eq!(account.lamports(), account_info.lamports()); - assert_eq!(account.data(), &account_info.data.borrow()[..]); - assert_eq!(account.owner(), account_info.owner); - assert_eq!(account.executable(), account_info.executable); - #[allow(deprecated)] - { - assert_eq!(u64::MAX, account_info._unused); - } - } + invoke_context.pop().unwrap(); + // check serialize_parameters_for_abiv0 + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 7, + instruction_accounts, + instruction_data.clone(), + ) + .unwrap(); + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); - deserialize_parameters( + let (serialized, regions, account_lengths, _instruction_data_offset) = + serialize_parameters( &instruction_context, - virtual_address_space_adjustments, - false, // account_data_direct_mapping - serialized.as_slice(), - &account_lengths, + direct_account_pointers_in_program_input, ) .unwrap(); - for (index_in_transaction, (_key, original_account)) in - original_accounts.iter().enumerate() + let mut serialized_regions = concat_regions(®ions); + + let (de_program_id, de_accounts, de_instruction_data) = unsafe { + deserialize_for_abiv0(serialized_regions.as_slice_mut().first_mut().unwrap() as *mut u8) + }; + assert_eq!(&program_id, de_program_id); + assert_eq!(instruction_data, de_instruction_data); + for account_info in de_accounts { + let index_in_transaction = invoke_context + .transaction_context + .find_index_of_account(account_info.key) + .unwrap(); + let account = invoke_context + .transaction_context + .accounts() + .try_borrow(index_in_transaction) + .unwrap(); + assert_eq!(account.lamports(), account_info.lamports()); + assert_eq!(account.data(), &account_info.data.borrow()[..]); + assert_eq!(account.owner(), account_info.owner); + assert_eq!(account.executable(), account_info.executable); + #[allow(deprecated)] { - let account = invoke_context - .transaction_context - .accounts() - .try_borrow(index_in_transaction as IndexOfAccount) - .unwrap(); - assert_eq!(&*account, original_account); + assert_eq!(u64::MAX, account_info._unused); } } + + deserialize_parameters( + &instruction_context, + serialized.as_slice(), + &account_lengths, + ) + .unwrap(); + for (index_in_transaction, (_key, original_account)) in original_accounts.iter().enumerate() + { + let account = invoke_context + .transaction_context + .accounts() + .try_borrow(index_in_transaction as IndexOfAccount) + .unwrap(); + assert_eq!(&*account, original_account); + } } #[test_case(false; "direct_account_pointers_in_program_input disabled")] @@ -1252,17 +1057,13 @@ mod tests { .configure_top_level_instruction_for_tests(0, instruction_accounts.clone(), vec![]) .unwrap(); invoke_context.push().unwrap(); - let instruction_context = invoke_context - .transaction_context - .get_current_instruction_context() - .unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); // check serialize_parameters_for_abiv1 let (_serialized, regions, _accounts_metadata, _instruction_data_offset) = serialize_parameters( &instruction_context, - true, - false, // account_data_direct_mapping direct_account_pointers_in_program_input, ) .unwrap(); @@ -1286,16 +1087,12 @@ mod tests { .configure_top_level_instruction_for_tests(7, instruction_accounts, vec![]) .unwrap(); invoke_context.push().unwrap(); - let instruction_context = invoke_context - .transaction_context - .get_current_instruction_context() - .unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); let (_serialized, regions, _account_lengths, _instruction_data_offset) = serialize_parameters( &instruction_context, - true, - false, // account_data_direct_mapping direct_account_pointers_in_program_input, ) .unwrap(); @@ -1313,6 +1110,12 @@ mod tests { } // the old bpf_loader in-program deserializer bpf_loader::id() + /// + /// # Safety + /// + /// `input` must point to a valid ABI-v0 serialized instruction buffer laid + /// out exactly as the legacy loader expects for the duration of the returned + /// borrows. #[deny(unsafe_op_in_unsafe_fn)] unsafe fn deserialize_for_abiv0<'a>( input: *mut u8, @@ -1326,11 +1129,7 @@ mod tests { fn read_possibly_unaligned(input: *mut u8, offset: usize) -> T { unsafe { let src = input.add(offset) as *const T; - if Self::COULD_BE_UNALIGNED { - src.read_unaligned() - } else { - src.read() - } + if Self::COULD_BE_UNALIGNED { src.read_unaligned() } else { src.read() } } } @@ -1450,6 +1249,273 @@ mod tests { mem } + fn write_vm_data_len( + serialized: &mut AlignedMemory, + account_metadata: &SerializedAccountMetadata, + len: usize, + ) { + let offset = account_metadata + .vm_data_addr + .saturating_sub(MM_INPUT_START) + .saturating_sub(size_of::() as u64) as usize; + serialized.as_slice_mut()[offset..offset + size_of::()] + .copy_from_slice(&(len as u64).to_le_bytes()); + } + + #[test] + fn test_vas_serialization_direct_maps_account_data_only() { + let program_id = Pubkey::new_unique(); + let account_data = b"direct-account-data-is-not-copied".to_vec(); + let transaction_accounts = vec![ + ( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + ), + ( + Pubkey::new_unique(), + AccountSharedData::from(Account { + lamports: 1, + data: account_data.clone(), + owner: program_id, + executable: false, + rent_epoch: 0, + }), + ), + ]; + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 0, + deduplicated_instruction_accounts(&[1], |_| true), + vec![], + ) + .unwrap(); + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + + let (serialized, regions, accounts_metadata, _instruction_data_offset) = + serialize_parameters(&instruction_context, false).unwrap(); + + assert!( + !serialized + .as_slice() + .windows(account_data.len()) + .any(|window| window == account_data) + ); + let data_region = regions + .iter() + .find(|region| region.vm_addr == accounts_metadata[0].vm_data_addr) + .unwrap(); + assert_eq!(data_region.len, account_data.len() as u64); + assert!(data_region.writable); + let mapped_data = unsafe { + slice::from_raw_parts(data_region.host_addr as *const u8, data_region.len as usize) + }; + assert_eq!(mapped_data, account_data); + } + + #[test_case(4, 8, Ok(4); "unchanged vm length restores original after transient growth")] + #[test_case(7, 8, Ok(7); "vm length grow wins over larger transient backing")] + #[test_case(2, 4, Ok(2); "vm length shrink truncates")] + #[test_case( + 4 + MAX_PERMITTED_DATA_INCREASE + 1, + 4, + Err(InstructionError::InvalidRealloc); + "invalid realloc limit errors" + )] + fn test_vas_deserialize_reconciles_direct_mapped_length( + vm_len: usize, + transient_len: usize, + expected: Result, + ) { + let program_id = Pubkey::new_unique(); + let transaction_accounts = vec![ + ( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + ), + ( + Pubkey::new_unique(), + AccountSharedData::from(Account { + lamports: 1, + data: vec![1, 2, 3, 4], + owner: program_id, + executable: false, + rent_epoch: 0, + }), + ), + ]; + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 0, + deduplicated_instruction_accounts(&[1], |_| true), + vec![], + ) + .unwrap(); + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + + let (mut serialized, _regions, accounts_metadata, _instruction_data_offset) = + serialize_parameters(&instruction_context, false).unwrap(); + write_vm_data_len(&mut serialized, &accounts_metadata[0], vm_len); + { + let mut account = instruction_context.try_borrow_instruction_account(0).unwrap(); + account.set_data_length(transient_len).unwrap(); + } + + let result = deserialize_parameters( + &instruction_context, + serialized.as_slice(), + &accounts_metadata, + ); + assert_eq!(result, expected.clone().map(|_| ())); + if let Ok(expected_len) = expected { + let account = instruction_context.try_borrow_instruction_account(0).unwrap(); + assert_eq!(account.get_data().len(), expected_len); + } + } + + #[test] + fn test_vas_borrowed_writable_account_store_uses_shadow_image() { + let program_id = Pubkey::new_unique(); + let initial_data = vec![1, 2, 3]; + let mut borrowed_buf = borrowed_account_buffer(initial_data.clone(), program_id); + let borrowed_account = borrowed_shared_data(&mut borrowed_buf); + let mut transaction_context = TransactionContext::new( + vec![ + (Pubkey::new_unique(), borrowed_account), + (program_id, AccountSharedData::default()), + ], + Rent::default(), + /* max_instruction_stack_depth */ 1, + /* max_instruction_trace_length */ 1, + /* number_of_top_level_instructions */ 1, + ); + transaction_context + .configure_top_level_instruction_for_tests( + 1, + vec![InstructionAccount::new(0, false, true)], + vec![], + ) + .unwrap(); + transaction_context.push().unwrap(); + let instruction_context = transaction_context.get_current_instruction_context().unwrap(); + let account_start_offset = MM_INPUT_START; + let region = create_memory_region_of_account( + &mut instruction_context.try_borrow_instruction_account(0).unwrap(), + account_start_offset, + ) + .unwrap(); + + assert_eq!(region.len, initial_data.len() as u64); + assert!(!region.writable); + assert_eq!(region.access_violation_handler_payload, Some(0)); + + let config = Config { + aligned_memory_mapping: false, + ..Config::default() + }; + let mut memory_mapping = unsafe { + MemoryMapping::new_with_access_violation_handler( + vec![region], + &config, + SBPFVersion::V3, + transaction_context.access_violation_handler(), + ) + } + .unwrap(); + + assert_eq!(memory_mapping.load::(account_start_offset).unwrap(), 1); + assert_eq!(active_borrowed_data(&mut borrowed_buf), initial_data); + + memory_mapping.store::(9, account_start_offset).unwrap(); + assert_eq!( + transaction_context.accounts().try_borrow(0).unwrap().data(), + &[9, 2, 3], + ); + assert_eq!(active_borrowed_data(&mut borrowed_buf), initial_data); + + { + let account = transaction_context.accounts().try_borrow(0).unwrap(); + match account.cow() { + CoWAccount::Borrowed(account) => account.commit(), + CoWAccount::Owned(_) => panic!("borrowed account should stay borrowed"), + } + } + assert_eq!(active_borrowed_data(&mut borrowed_buf), vec![9, 2, 3]); + } + + #[test] + fn test_vas_borrowed_writable_account_store_without_commit_keeps_active_image() { + let program_id = Pubkey::new_unique(); + let initial_data = vec![4, 5, 6]; + let mut borrowed_buf = borrowed_account_buffer(initial_data.clone(), program_id); + let borrowed_account = borrowed_shared_data(&mut borrowed_buf); + let mut transaction_context = TransactionContext::new( + vec![ + (Pubkey::new_unique(), borrowed_account), + (program_id, AccountSharedData::default()), + ], + Rent::default(), + /* max_instruction_stack_depth */ 1, + /* max_instruction_trace_length */ 1, + /* number_of_top_level_instructions */ 1, + ); + transaction_context + .configure_top_level_instruction_for_tests( + 1, + vec![InstructionAccount::new(0, false, true)], + vec![], + ) + .unwrap(); + transaction_context.push().unwrap(); + let instruction_context = transaction_context.get_current_instruction_context().unwrap(); + let account_start_offset = MM_INPUT_START; + let region = create_memory_region_of_account( + &mut instruction_context.try_borrow_instruction_account(0).unwrap(), + account_start_offset, + ) + .unwrap(); + let config = Config { + aligned_memory_mapping: false, + ..Config::default() + }; + let mut memory_mapping = unsafe { + MemoryMapping::new_with_access_violation_handler( + vec![region], + &config, + SBPFVersion::V3, + transaction_context.access_violation_handler(), + ) + } + .unwrap(); + + memory_mapping.store::(7, account_start_offset).unwrap(); + + assert_eq!( + transaction_context.accounts().try_borrow(0).unwrap().data(), + &[7, 5, 6], + ); + assert_eq!(active_borrowed_data(&mut borrowed_buf), initial_data); + } + #[test] fn test_access_violation_handler() { let program_id = Pubkey::new_unique(); @@ -1481,6 +1547,10 @@ mod tests { Pubkey::new_unique(), AccountSharedData::new(0, 0x3000, &program_id), ), // writable dummy to burn accounts_resize_delta + ( + Pubkey::new_unique(), + AccountSharedData::new(0, 0, &program_id), + ), // writable dummy to burn accounts_resize_delta (program_id, AccountSharedData::default()), // program ], Rent::default(), @@ -1488,16 +1558,14 @@ mod tests { /* max_instruction_trace_length */ 1, /* number_of_top_level_instructions */ 1, ); - let transaction_accounts_indexes = [0, 1, 2, 3, 4, 5]; + let transaction_accounts_indexes = [0, 1, 2, 3, 4, 5, 6]; let instruction_accounts = deduplicated_instruction_accounts(&transaction_accounts_indexes, |index| index > 0); transaction_context - .configure_top_level_instruction_for_tests(6, instruction_accounts, vec![]) + .configure_top_level_instruction_for_tests(7, instruction_accounts, vec![]) .unwrap(); transaction_context.push().unwrap(); - let instruction_context = transaction_context - .get_current_instruction_context() - .unwrap(); + let instruction_context = transaction_context.get_current_instruction_context().unwrap(); let account_start_offsets = [ MM_INPUT_START, MM_INPUT_START + 4 + MAX_PERMITTED_DATA_INCREASE as u64, @@ -1526,90 +1594,46 @@ mod tests { regions, &config, SBPFVersion::V3, - transaction_context.access_violation_handler(true, true), + transaction_context.access_violation_handler(), ) - .unwrap() - }; + } + .unwrap(); // Reading readonly account is allowed - memory_mapping - .load::(account_start_offsets[0]) - .unwrap(); + memory_mapping.load::(account_start_offsets[0]).unwrap(); // Reading writable account is allowed - memory_mapping - .load::(account_start_offsets[1]) - .unwrap(); + memory_mapping.load::(account_start_offsets[1]).unwrap(); // Reading beyond readonly accounts current size is denied - memory_mapping - .load::(account_start_offsets[0] + 4) - .unwrap_err(); + memory_mapping.load::(account_start_offsets[0] + 4).unwrap_err(); // Writing to readonly account is denied - memory_mapping - .store::(0, account_start_offsets[0]) - .unwrap_err(); + memory_mapping.store::(0, account_start_offsets[0]).unwrap_err(); // Writing to shared writable account makes it unique (CoW logic.) // It has been previously been made non-unique at the beginning of // the test through a clone. let _shared_account_ref = shared_account; - assert!( - transaction_context - .accounts() - .try_borrow_mut(1) - .unwrap() - .is_shared() - ); - memory_mapping - .store::(0, account_start_offsets[1]) - .unwrap(); - assert!( - !transaction_context - .accounts() - .try_borrow_mut(1) - .unwrap() - .is_shared() - ); + assert!(transaction_context.accounts().try_borrow_mut(1).unwrap().is_shared()); + memory_mapping.store::(0, account_start_offsets[1]).unwrap(); + assert!(!transaction_context.accounts().try_borrow_mut(1).unwrap().is_shared()); assert_eq!( - transaction_context - .accounts() - .try_borrow(1) - .unwrap() - .data() - .len(), + transaction_context.accounts().try_borrow(1).unwrap().data().len(), 4, ); // Reading beyond writable accounts current size grows is denied - memory_mapping - .load::(account_start_offsets[1] + 4) - .unwrap_err(); + memory_mapping.load::(account_start_offsets[1] + 4).unwrap_err(); - // Writing beyond writable accounts current size grows it - // to original length plus MAX_PERMITTED_DATA_INCREASE - memory_mapping - .store::(0, account_start_offsets[1] + 4) - .unwrap(); + // Writing beyond writable accounts current size grows it only to the + // requested access length. + memory_mapping.store::(0, account_start_offsets[1] + 4).unwrap(); assert_eq!( - transaction_context - .accounts() - .try_borrow(1) - .unwrap() - .data() - .len(), - 4 + MAX_PERMITTED_DATA_INCREASE, - ); - assert!( - transaction_context - .accounts() - .try_borrow(1) - .unwrap() - .data() - .len() - < 0x3000 + transaction_context.accounts().try_borrow(1).unwrap().data().len(), + 8, ); + assert!(transaction_context.accounts().try_borrow(1).unwrap().data().len() < 0x3000); // Writing beyond almost max sized writable accounts current size only grows it // to MAX_PERMITTED_DATA_LENGTH @@ -1617,12 +1641,7 @@ mod tests { .store::(0, account_start_offsets[3] + MAX_PERMITTED_DATA_LENGTH - 4) .unwrap(); assert_eq!( - transaction_context - .accounts() - .try_borrow(3) - .unwrap() - .data() - .len(), + transaction_context.accounts().try_borrow(3).unwrap().data().len(), MAX_PERMITTED_DATA_LENGTH as usize, ); @@ -1637,33 +1656,41 @@ mod tests { // Burn through most of the accounts_resize_delta budget let remaining_allowed_growth: usize = 0x700; - for index_in_instruction in 4..6 { + let target_resize_delta = MAX_PERMITTED_ACCOUNTS_DATA_ALLOCATIONS_PER_TRANSACTION + - remaining_allowed_growth as i64; + for index_in_instruction in 4..7 { + let burn = target_resize_delta + .saturating_sub(transaction_context.accounts().resize_delta()) + as usize; + if burn == 0 { + break; + } let mut borrowed_account = instruction_context .try_borrow_instruction_account(index_in_instruction) .unwrap(); - borrowed_account - .set_data_from_slice(&vec![0u8; MAX_PERMITTED_DATA_LENGTH as usize]) - .unwrap(); + let old_len = borrowed_account.get_data().len(); + let new_len = old_len.saturating_add(burn).min(MAX_PERMITTED_DATA_LENGTH as usize); + borrowed_account.set_data_length(new_len).unwrap(); } assert_eq!( transaction_context.accounts().resize_delta(), - MAX_PERMITTED_ACCOUNTS_DATA_ALLOCATIONS_PER_TRANSACTION - - remaining_allowed_growth as i64, + target_resize_delta, ); - // Writing beyond empty writable accounts current size - // only grows it to fill up MAX_PERMITTED_ACCOUNTS_DATA_ALLOCATIONS_PER_TRANSACTION - memory_mapping - .store::(0, account_start_offsets[2] + 0x500) - .unwrap(); + // Writing beyond empty writable accounts current size grows to the + // requested access length while it fits in the remaining transaction budget. + memory_mapping.store::(0, account_start_offsets[2] + 0x500).unwrap(); assert_eq!( - transaction_context - .accounts() - .try_borrow(2) - .unwrap() - .data() - .len(), - remaining_allowed_growth, + transaction_context.accounts().try_borrow(2).unwrap().data().len(), + 0x504, ); + + // A write that would need more than the remaining transaction budget is denied. + memory_mapping + .store::( + 0, + account_start_offsets[2] + remaining_allowed_growth as u64, + ) + .unwrap_err(); } } diff --git a/solana/program-runtime/src/sysvar_cache.rs b/solana/program-runtime/src/sysvar_cache.rs index 8d018cd..2277c68 100644 --- a/solana/program-runtime/src/sysvar_cache.rs +++ b/solana/program-runtime/src/sysvar_cache.rs @@ -1,124 +1,51 @@ -#[expect(deprecated)] -use solana_sysvar::{fees::Fees, recent_blockhashes::RecentBlockhashes}; +use solana_epoch_rewards::EpochRewards; +#[allow(deprecated)] +use solana_sysvar::fees::Fees; +#[allow(deprecated)] +use solana_sysvar::recent_blockhashes::RecentBlockhashes; use { crate::invoke_context::InvokeContext, serde::de::DeserializeOwned, solana_clock::Clock, - solana_epoch_rewards::EpochRewards, solana_epoch_schedule::EpochSchedule, solana_instruction::error::InstructionError, solana_last_restart_slot::LastRestartSlot, solana_pubkey::Pubkey, solana_rent::Rent, - solana_sdk_ids::sysvar, solana_slot_hashes::SlotHashes, - solana_stake_interface::stake_history::StakeHistory, solana_svm_type_overrides::sync::Arc, - solana_sysvar::SysvarSerialize, solana_sysvar_id::SysvarId, solana_transaction_context::{IndexOfAccount, instruction::InstructionContext}, }; -#[cfg(feature = "frozen-abi")] -impl ::solana_frozen_abi::abi_example::AbiExample for SysvarCache { - fn example() -> Self { - // SysvarCache is not Serialize so just rely on Default. - SysvarCache::default() - } -} - -#[derive(Default, Clone, Debug)] +/// Serialized sysvars exposed to programs during execution. +#[derive(Default, Debug)] pub struct SysvarCache { - // full account data as provided by bank, including any trailing zero bytes + // Full account data, including any trailing zero bytes. clock: Option>, epoch_schedule: Option>, - epoch_rewards: Option>, rent: Option>, slot_hashes: Option>, - stake_history: Option>, last_restart_slot: Option>, - // object representations of large sysvars for convenience - // these are used by the stake and vote builtin programs - // these should be removed once those programs are ported to bpf - slot_hashes_obj: Option>, - stake_history_obj: Option>, + // Object representations of large sysvars used by native builtins. + slot_hashes_obj: Option, - // deprecated sysvars, these should be removed once practical - #[expect(deprecated)] - fees: Option, - #[expect(deprecated)] + #[allow(deprecated)] recent_blockhashes: Option, } -// declare_deprecated_sysvar_id doesn't support const. -// These sysvars are going away anyway. -const FEES_ID: Pubkey = Pubkey::from_str_const("SysvarFees111111111111111111111111111111111"); -const RECENT_BLOCKHASHES_ID: Pubkey = - Pubkey::from_str_const("SysvarRecentB1ockHashes11111111111111111111"); - impl SysvarCache { - /// Overwrite a sysvar. For testing purposes only. - #[expect(deprecated)] - pub fn set_sysvar_for_tests(&mut self, sysvar: &T) { - let data = bincode::serialize(sysvar).expect("Failed to serialize sysvar."); - let sysvar_id = T::id(); - match sysvar_id { - sysvar::clock::ID => { - self.clock = Some(data); - } - sysvar::epoch_rewards::ID => { - self.epoch_rewards = Some(data); - } - sysvar::epoch_schedule::ID => { - self.epoch_schedule = Some(data); - } - FEES_ID => { - let fees: Fees = - bincode::deserialize(&data).expect("Failed to deserialize Fees sysvar."); - self.fees = Some(fees); - } - sysvar::last_restart_slot::ID => { - self.last_restart_slot = Some(data); - } - RECENT_BLOCKHASHES_ID => { - let recent_blockhashes: RecentBlockhashes = bincode::deserialize(&data) - .expect("Failed to deserialize RecentBlockhashes sysvar."); - self.recent_blockhashes = Some(recent_blockhashes); - } - sysvar::rent::ID => { - self.rent = Some(data); - } - sysvar::slot_hashes::ID => { - let slot_hashes: SlotHashes = - bincode::deserialize(&data).expect("Failed to deserialize SlotHashes sysvar."); - self.slot_hashes = Some(data); - self.slot_hashes_obj = Some(Arc::new(slot_hashes)); - } - sysvar::stake_history::ID => { - let stake_history: StakeHistory = bincode::deserialize(&data) - .expect("Failed to deserialize StakeHistory sysvar."); - self.stake_history = Some(data); - self.stake_history_obj = Some(Arc::new(stake_history)); - } - _ => panic!("Unrecognized Sysvar ID: {sysvar_id}"), - } - } - - // this is exposed for SyscallGetSysvar and should not otherwise be used + /// Returns the serialized sysvar buffer for `SyscallGetSysvar`. pub fn sysvar_id_to_buffer(&self, sysvar_id: &Pubkey) -> &Option> { if Clock::check_id(sysvar_id) { &self.clock } else if EpochSchedule::check_id(sysvar_id) { &self.epoch_schedule - } else if EpochRewards::check_id(sysvar_id) { - &self.epoch_rewards } else if Rent::check_id(sysvar_id) { &self.rent } else if SlotHashes::check_id(sysvar_id) { &self.slot_hashes - } else if StakeHistory::check_id(sysvar_id) { - &self.stake_history } else if LastRestartSlot::check_id(sysvar_id) { &self.last_restart_slot } else { @@ -126,8 +53,6 @@ impl SysvarCache { } } - // most if not all of the obj getter functions can be removed once builtins transition to bpf - // the Arc wrapper is to preserve the existing public interface fn get_sysvar_obj( &self, sysvar_id: &Pubkey, @@ -141,49 +66,40 @@ impl SysvarCache { } } - pub fn get_clock(&self) -> Result, InstructionError> { - self.get_sysvar_obj(&Clock::id()) - } - - pub fn get_epoch_schedule(&self) -> Result, InstructionError> { - self.get_sysvar_obj(&EpochSchedule::id()) + /// Stores a serialized clock sysvar. + pub fn set_clock(&mut self, clock: &Clock) { + let buffer = self.clock.get_or_insert_default(); + buffer.clear(); + // bincode doesn't fail when writing to a correctly sized sysvar buffer. + let _ = bincode::serialize_into(buffer, clock); } - pub fn get_epoch_rewards(&self) -> Result, InstructionError> { - self.get_sysvar_obj(&EpochRewards::id()) + /// Returns the cached clock sysvar. + pub fn get_clock(&self) -> Result, InstructionError> { + self.get_sysvar_obj(&Clock::id()) } + /// Returns the cached rent sysvar. pub fn get_rent(&self) -> Result, InstructionError> { self.get_sysvar_obj(&Rent::id()) } + /// Returns the cached last-restart-slot sysvar. pub fn get_last_restart_slot(&self) -> Result, InstructionError> { self.get_sysvar_obj(&LastRestartSlot::id()) } - pub fn get_stake_history(&self) -> Result, InstructionError> { - self.stake_history_obj - .clone() - .ok_or(InstructionError::UnsupportedSysvar) - } - + /// Returns the cached slot hashes sysvar. pub fn get_slot_hashes(&self) -> Result, InstructionError> { self.slot_hashes_obj - .clone() - .ok_or(InstructionError::UnsupportedSysvar) - } - - #[deprecated] - #[expect(deprecated)] - pub fn get_fees(&self) -> Result, InstructionError> { - self.fees - .clone() + .as_ref() + .map(|s| Arc::new(SlotHashes::new(s.slot_hashes()))) .ok_or(InstructionError::UnsupportedSysvar) - .map(Arc::new) } #[deprecated] - #[expect(deprecated)] + #[allow(deprecated)] + /// Returns the cached recent-blockhashes sysvar. pub fn get_recent_blockhashes(&self) -> Result, InstructionError> { self.recent_blockhashes .clone() @@ -191,6 +107,23 @@ impl SysvarCache { .map(Arc::new) } + /// Returns the (deprecated) fees sysvar; this engine always reports defaults. + #[allow(deprecated)] + pub fn get_fees(&self) -> Result, InstructionError> { + Ok(Arc::new(Default::default())) + } + + /// Returns the epoch-schedule sysvar; this engine always reports defaults. + pub fn get_epoch_schedule(&self) -> Result, InstructionError> { + Ok(Arc::new(Default::default())) + } + + /// Returns the epoch-rewards sysvar; this engine always reports defaults. + pub fn get_epoch_rewards(&self) -> Result, InstructionError> { + Ok(Arc::new(Default::default())) + } + + /// Fills missing sysvars by asking the caller for serialized account data. pub fn fill_missing_entries( &mut self, mut get_account_data: F, @@ -211,14 +144,6 @@ impl SysvarCache { }); } - if self.epoch_rewards.is_none() { - get_account_data(&EpochRewards::id(), &mut |data: &[u8]| { - if bincode::deserialize::(data).is_ok() { - self.epoch_rewards = Some(data.to_vec()); - } - }); - } - if self.rent.is_none() { get_account_data(&Rent::id(), &mut |data: &[u8]| { if bincode::deserialize::(data).is_ok() { @@ -231,16 +156,7 @@ impl SysvarCache { get_account_data(&SlotHashes::id(), &mut |data: &[u8]| { if let Ok(obj) = bincode::deserialize::(data) { self.slot_hashes = Some(data.to_vec()); - self.slot_hashes_obj = Some(Arc::new(obj)); - } - }); - } - - if self.stake_history.is_none() { - get_account_data(&StakeHistory::id(), &mut |data: &[u8]| { - if let Ok(obj) = bincode::deserialize::(data) { - self.stake_history = Some(data.to_vec()); - self.stake_history_obj = Some(Arc::new(obj)); + self.slot_hashes_obj = Some(obj); } }); } @@ -253,16 +169,7 @@ impl SysvarCache { }); } - #[expect(deprecated)] - if self.fees.is_none() { - get_account_data(&Fees::id(), &mut |data: &[u8]| { - if let Ok(fees) = bincode::deserialize(data) { - self.fees = Some(fees); - } - }); - } - - #[expect(deprecated)] + #[allow(deprecated)] if self.recent_blockhashes.is_none() { get_account_data(&RecentBlockhashes::id(), &mut |data: &[u8]| { if let Ok(recent_blockhashes) = bincode::deserialize(data) { @@ -272,15 +179,14 @@ impl SysvarCache { } } + /// Clears all cached sysvars. pub fn reset(&mut self) { *self = Self::default(); } } -/// These methods facilitate a transition from fetching sysvars from keyed -/// accounts to fetching from the sysvar cache without breaking consensus. In -/// order to keep consistent behavior, they continue to enforce legacy checks -/// despite dynamically loading them instead of deserializing from account data. +/// Sysvar accessors that also verify the instruction account matches the +/// requested sysvar id. pub mod get_sysvar_with_account_check { use super::*; @@ -296,59 +202,45 @@ pub mod get_sysvar_with_account_check { Ok(()) } + /// Returns the clock sysvar after checking the provided instruction account. pub fn clock( invoke_context: &InvokeContext, instruction_context: &InstructionContext, instruction_account_index: IndexOfAccount, ) -> Result, InstructionError> { check_sysvar_account::(instruction_context, instruction_account_index)?; - invoke_context.environment_config.sysvar_cache().get_clock() + invoke_context.get_sysvar_cache().get_clock() } + /// Returns the rent sysvar after checking the provided instruction account. pub fn rent( invoke_context: &InvokeContext, instruction_context: &InstructionContext, instruction_account_index: IndexOfAccount, ) -> Result, InstructionError> { check_sysvar_account::(instruction_context, instruction_account_index)?; - invoke_context.environment_config.sysvar_cache().get_rent() + invoke_context.get_sysvar_cache().get_rent() } + /// Returns slot hashes after checking the provided instruction account. pub fn slot_hashes( invoke_context: &InvokeContext, instruction_context: &InstructionContext, instruction_account_index: IndexOfAccount, ) -> Result, InstructionError> { check_sysvar_account::(instruction_context, instruction_account_index)?; - invoke_context - .environment_config - .sysvar_cache() - .get_slot_hashes() + invoke_context.get_sysvar_cache().get_slot_hashes() } - #[expect(deprecated)] + #[allow(deprecated)] + /// Returns recent blockhashes after checking the provided instruction account. pub fn recent_blockhashes( invoke_context: &InvokeContext, instruction_context: &InstructionContext, instruction_account_index: IndexOfAccount, ) -> Result, InstructionError> { check_sysvar_account::(instruction_context, instruction_account_index)?; - invoke_context - .environment_config - .sysvar_cache() - .get_recent_blockhashes() - } - - pub fn stake_history( - invoke_context: &InvokeContext, - instruction_context: &InstructionContext, - instruction_account_index: IndexOfAccount, - ) -> Result, InstructionError> { - check_sysvar_account::(instruction_context, instruction_account_index)?; - invoke_context - .environment_config - .sysvar_cache() - .get_stake_history() + invoke_context.get_sysvar_cache().get_recent_blockhashes() } pub fn last_restart_slot( @@ -357,16 +249,13 @@ pub mod get_sysvar_with_account_check { instruction_account_index: IndexOfAccount, ) -> Result, InstructionError> { check_sysvar_account::(instruction_context, instruction_account_index)?; - invoke_context - .environment_config - .sysvar_cache() - .get_last_restart_slot() + invoke_context.get_sysvar_cache().get_last_restart_slot() } } #[cfg(test)] mod tests { - use {super::*, test_case::test_case}; + use {super::*, solana_sysvar::SysvarSerialize, test_case::test_case}; // sysvar cache provides the full account data of a sysvar // the setters MUST NOT be changed to serialize an object representation @@ -376,11 +265,8 @@ mod tests { // * vector sysvar has fewer than its maximum entries // if at any point the data is roundtripped through bincode, the vector will shrink #[test_case(Clock::default(); "clock")] - #[test_case(EpochSchedule::default(); "epoch_schedule")] - #[test_case(EpochRewards::default(); "epoch_rewards")] #[test_case(Rent::default(); "rent")] #[test_case(SlotHashes::default(); "slot_hashes")] - #[test_case(StakeHistory::default(); "stake_history")] #[test_case(LastRestartSlot::default(); "last_restart_slot")] fn test_sysvar_cache_preserves_bytes(_: T) { let id = T::id(); diff --git a/solana/program-runtime/src/vm.rs b/solana/program-runtime/src/vm.rs index 119c6f0..0e83a10 100644 --- a/solana/program-runtime/src/vm.rs +++ b/solana/program-runtime/src/vm.rs @@ -4,34 +4,34 @@ use qualifier_attr::qualifiers; use { crate::{ - execution_budget::MAX_INSTRUCTION_STACK_DEPTH_SIMD_0268, - invoke_context::{BpfAllocator, InvokeContext}, + execution_budget::MAX_INSTRUCTION_STACK_DEPTH, + invoke_context::{BpfAllocator, InvokeContext, SerializedAccountMetadata, SyscallContext}, mem_pool::VmMemoryPool, - memory_context::{MemoryContext, SerializedAccountMetadata}, - program_cache_entry::ProgramCacheEntry, + memory_context::MemoryContext, serialization, stable_log, }, + cfg_if::cfg_if, solana_instruction::error::InstructionError, solana_program_entrypoint::{MAX_PERMITTED_DATA_INCREASE, SUCCESS}, solana_sbpf::{ - ebpf::{self, MM_HEAP_START, MM_STACK_START}, + ebpf::{self, MM_HEAP_START}, elf::Executable, error::{EbpfError, ProgramResult}, memory_region::{AccessType, MemoryMapping, MemoryRegion}, - vm::{ContextObject, EbpfVm, ExecutionMode}, + vm::{CallFrame, ContextObject, EbpfVm, ExecutionMode}, }, solana_sdk_ids::bpf_loader_deprecated, solana_svm_log_collector::ic_logger_msg, solana_svm_measure::measure::Measure, - solana_transaction_context::IndexOfAccount, - std::{cell::RefCell, mem, time::Duration}, + solana_transaction_context::{IndexOfAccount, transaction::TransactionContext}, + std::{cell::RefCell, mem}, }; thread_local! { pub static MEMORY_POOL: RefCell = RefCell::new(VmMemoryPool::new()); } -/// Only used in macro, do not use directly! +/// Calculates extra compute units charged for a requested heap size. pub fn calculate_heap_cost(heap_size: u32, heap_cost: u64) -> u64 { const KIBIBYTE: u64 = 1024; const PAGE_SIZE_KB: u64 = 32; @@ -45,23 +45,34 @@ pub fn calculate_heap_cost(heap_size: u32, heap_cost: u64) -> u64 { .saturating_mul(heap_cost) } -/// Only used in macro, do not use directly! -/// -/// # Safety -/// -/// Refer to [`configure_program_regions`]. +/// Creates an SBF VM bound to the current invocation context. #[cfg_attr(feature = "svm-internal", qualifiers(pub))] -pub unsafe fn create_vm<'a, 'b>( - program: &'a Executable>, - invoke_context: &'a mut InvokeContext<'b, 'b>, - stack: *mut [u8], - heap: *mut [u8], -) -> Result>, Box> { +pub fn create_vm<'a, 'b, 'c>( + program: &'a Executable>, + regions: Vec, + accounts_metadata: Vec, + invoke_context: &'a mut InvokeContext<'b, 'c>, + stack: &mut [u8], + heap: &mut [u8], +) -> Result>, Box> { let stack_size = stack.len(); - unsafe { - // SAFETY: invariants delegated to the caller. - configure_program_regions(invoke_context, program, stack, heap)?; - } + let heap_size = heap.len(); + let memory_mapping = create_memory_mapping( + program, + stack, + heap, + regions, + invoke_context.transaction_context, + )?; + invoke_context.set_syscall_context(SyscallContext { + allocator: BpfAllocator::new(heap_size as u64), + accounts_metadata: accounts_metadata.clone(), + })?; + invoke_context.memory_contexts.set_memory_context_abi_v1(MemoryContext::new( + BpfAllocator::new(heap_size as u64), + accounts_metadata, + memory_mapping, + ))?; Ok(EbpfVm::new( program.get_loader().clone(), program.get_sbpf_version(), @@ -70,126 +81,81 @@ pub unsafe fn create_vm<'a, 'b>( )) } -/// # Safety -/// -/// The `executable`, `stack` and `heap` arguments must remain allocated for at least the lifetime -/// of [`MemoryMapping`] (or until after the `MemoryMapping` is reconfigured with different -/// `executable`, `stack` and `heap`). -unsafe fn configure_program_regions( - invoke_context: &mut InvokeContext, +fn create_memory_mapping<'a, C: ContextObject>( executable: &Executable, - stack: *mut [u8], - heap: *mut [u8], -) -> Result<(), Box> { - let mapping = invoke_context.memory_contexts.memory_mapping_mut()?; - let regions = mapping.get_regions_mut(); - let [ro_area, stack_area, heap_area, ..] = regions else { - panic!("the regions vector must have at least three entries") - }; - *ro_area = executable.get_ro_region(); - let sbpf_version = executable.get_sbpf_version(); + stack: &'a mut [u8], + heap: &'a mut [u8], + additional_regions: Vec, + transaction_context: &TransactionContext, +) -> Result> { let config = executable.get_config(); - *stack_area = MemoryRegion::new_gapped( - stack, - MM_STACK_START, - if sbpf_version.stack_frame_gaps() && config.enable_stack_frame_gaps { - config.stack_frame_size as u64 - } else { - 0 - }, - ); - *heap_area = MemoryRegion::new(heap, MM_HEAP_START); - mapping - .initialize() - .map_err(|err| Box::new(err) as Box) + let sbpf_version = executable.get_sbpf_version(); + let regions: Vec = vec![ + executable.get_ro_region(), + MemoryRegion::new_gapped( + &raw mut stack[..], + ebpf::MM_STACK_START, + if sbpf_version.stack_frame_gaps() && config.enable_stack_frame_gaps { + config.stack_frame_size as u64 + } else { + 0 + }, + ), + MemoryRegion::new(&raw mut heap[..], MM_HEAP_START), + ] + .into_iter() + .chain(additional_regions) + .collect(); + + Ok(unsafe { + MemoryMapping::new_with_access_violation_handler( + regions, + config, + sbpf_version, + transaction_context.access_violation_handler(), + )? + }) } /// Create the SBF virtual machine #[macro_export] macro_rules! create_vm { - ($vm:ident, $program:expr, $invoke_context:expr $(,)?) => { + ($vm:ident, $program:expr, $regions:expr, $accounts_metadata:expr, $invoke_context:expr $(,)?) => { let invoke_context = &*$invoke_context; let stack_size = $program.get_config().stack_size(); let heap_size = invoke_context.get_compute_budget().heap_size; let heap_cost_result = - invoke_context - .compute_meter - .consume_checked($crate::__private::calculate_heap_cost( - heap_size, - invoke_context.get_execution_cost().heap_cost, - )); + invoke_context.consume_checked($crate::__private::calculate_heap_cost( + heap_size, + invoke_context.get_execution_cost().heap_cost, + )); let $vm = heap_cost_result.and_then(|_| { let (mut stack, mut heap) = $crate::__private::MEMORY_POOL .with_borrow_mut(|pool| (pool.get_stack(stack_size), pool.get_heap(heap_size))); let vm = $crate::__private::create_vm( $program, + $regions, + $accounts_metadata, $invoke_context, - stack - .as_slice_mut() - .get_mut(..stack_size) - .expect("invalid stack size"), - heap.as_slice_mut() - .get_mut(..heap_size as usize) - .expect("invalid heap size"), + stack.as_slice_mut().get_mut(..stack_size).expect("invalid stack size"), + heap.as_slice_mut().get_mut(..heap_size as usize).expect("invalid heap size"), ); vm.map(|vm| (vm, stack, heap)) }); }; } -/// # Safety -/// -/// The [`MemoryRegion`]s must satisfy the safety preconditions for -/// [`MemoryMapping::new_uninitialized`]. -unsafe fn set_memory_context<'b>( - additional_initialized_regions: Vec, - accounts_metadata: Vec, - invoke_context: &mut InvokeContext<'b, 'b>, - executable: &Executable>, - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, -) -> Result<(), Box> { - let heap_size = invoke_context.get_compute_budget().heap_size; - let regions = vec![MemoryRegion::default(); 3] - .into_iter() - .chain(additional_initialized_regions) - .collect(); - let memory_mapping = unsafe { - // SAFETY: all memory regions are `default` (and thus implicitly valid) or valid by - // delegating the safety invariant upon the caller. - MemoryMapping::new_uninitialized( - regions, - executable.get_config(), - executable.get_sbpf_version(), - invoke_context.transaction_context.access_violation_handler( - virtual_address_space_adjustments, - account_data_direct_mapping, - ), - ) - }; - - invoke_context - .memory_contexts - .set_memory_context_abi_v1(MemoryContext::new( - BpfAllocator::new(heap_size as u64), - accounts_metadata, - memory_mapping, - )) - .map_err(|err| Box::new(err) as Box) -} - #[cfg_attr(feature = "svm-internal", qualifiers(pub))] -pub fn execute<'a, 'b: 'a>( +pub fn execute<'a, 'b, 'c>( executable: &'a Executable>, - invoke_context: &'a mut InvokeContext<'b, 'b>, - cache_entry: &ProgramCacheEntry, + invoke_context: &'a mut InvokeContext<'b, 'c>, ) -> Result<(), Box> { // We dropped the lifetime tracking in the Executor by setting it to 'static, // thus we need to reintroduce the correct lifetime of InvokeContext here again. let executable = unsafe { mem::transmute::< &'a Executable>, - &'a Executable>, + &'a Executable>, >(executable) }; let log_collector = invoke_context.get_log_collector(); @@ -198,20 +164,41 @@ pub fn execute<'a, 'b: 'a>( let program_id = *instruction_context.get_program_key()?; let is_loader_deprecated = instruction_context.get_program_owner()? == bpf_loader_deprecated::id(); - let virtual_address_space_adjustments = invoke_context - .get_feature_set() - .virtual_address_space_adjustments; - let account_data_direct_mapping = invoke_context.get_feature_set().account_data_direct_mapping; - let direct_account_pointers_in_program_input = invoke_context - .get_feature_set() - .direct_account_pointers_in_program_input; + cfg_if! { + if #[cfg(any( + target_os = "windows", + not(target_arch = "x86_64"), + feature = "sbpf-debugger" + ))] { + let use_jit = false; + #[cfg(feature = "sbpf-debugger")] + let debug_metadata = format!( + "program_id={};cpi_level={};caller={}", + program_id, + instruction_context.get_stack_height().saturating_sub(1), + invoke_context + .get_stack_height() + .checked_sub(2) + .and_then(|nesting_level| { + transaction_context + .get_instruction_context_at_nesting_level(nesting_level) + .ok() + }) + .and_then(|ctx| ctx.get_program_key().ok()) + .map(|key| key.to_string()) + .unwrap_or_else(|| "none".into()) + ); + } else { + let use_jit = executable.get_compiled_program().is_some(); + } + } + let direct_account_pointers_in_program_input = + invoke_context.get_feature_set().direct_account_pointers_in_program_input; let mut serialize_time = Measure::start("serialize"); let (parameter_bytes, regions, accounts_metadata, instruction_data_offset) = serialization::serialize_parameters( &instruction_context, - virtual_address_space_adjustments, - account_data_direct_mapping, direct_account_pointers_in_program_input, )?; serialize_time.stop(); @@ -221,123 +208,55 @@ pub fn execute<'a, 'b: 'a>( let account_region_addrs = accounts_metadata .iter() .map(|m| { - let vm_end = m - .vm_data_addr - .saturating_add(m.original_data_len as u64) - .saturating_add(if !is_loader_deprecated { - MAX_PERMITTED_DATA_INCREASE as u64 - } else { - 0 - }); + let vm_end = m.vm_data_addr.saturating_add(m.original_data_len as u64).saturating_add( + if !is_loader_deprecated { MAX_PERMITTED_DATA_INCREASE as u64 } else { 0 }, + ); m.vm_data_addr..vm_end }) .collect::>(); - #[cfg(feature = "sbpf-debugger")] - let (debug_port, debug_metadata) = if invoke_context.debug_port.is_some() { - ( - invoke_context.debug_port, - Some(format!( - "program_id={};cpi_level={};caller={}", - program_id, - instruction_context.get_stack_height().saturating_sub(1), - invoke_context - .get_stack_height() - .checked_sub(2) - .and_then(|nesting_level| { - transaction_context - .get_instruction_context_at_nesting_level(nesting_level) - .ok() - }) - .and_then(|ctx| ctx.get_program_key().ok()) - .map(|key| key.to_string()) - .unwrap_or_else(|| "none".into()) - )), - ) - } else { - (None, None) - }; - let mut create_vm_time = Measure::start("create_vm"); - unsafe { - // SAFETY: The memory pointed to by regions is valid for the useful lifetime of - // `invoke_context`, which in turn contains the `MemoryMapping` that allows access to this - // memory. - set_memory_context( - regions, - accounts_metadata, - invoke_context, - executable, - virtual_address_space_adjustments, - account_data_direct_mapping, - )? - }; - let execution_result = { - let mut execution_mode = ExecutionMode::PreferJit; - - #[cfg(feature = "sbpf-debugger")] - if invoke_context.debug_port.is_some() { - execution_mode = ExecutionMode::Interpreted; - } - let compute_meter_prev = invoke_context.get_remaining(); - let (mut vm, stack, heap) = unsafe { - // SAFETY: The `stack`, `heap` and `executable` live past the lifetime of - // `invoke_context`. - create_vm!(vm, executable, invoke_context); - match vm { - Ok(info) => info, - Err(e) => { - ic_logger_msg!(log_collector, "Failed to create SBF VM: {}", e); - return Err(Box::new(InstructionError::ProgramEnvironmentSetupFailure)); - } + create_vm!(vm, executable, regions, accounts_metadata, invoke_context); + let (mut vm, stack, heap) = match vm { + Ok(info) => info, + Err(e) => { + ic_logger_msg!(log_collector, "Failed to create SBF VM: {}", e); + return Err(Box::new(InstructionError::ProgramEnvironmentSetupFailure)); } }; - create_vm_time.stop(); + #[cfg(feature = "sbpf-debugger")] { - vm.debug_metadata = debug_metadata; + vm.debug_metadata = Some(debug_metadata); } - - let execute_time = Measure::start("execute"); - let prev_nested_exec_time = vm.context().total_nested_exec_time; - + let mut execute_time = Measure::start("execute"); vm.registers[1] = ebpf::MM_INPUT_START; vm.registers[2] = instruction_data_offset as u64; - let mut call_frames = - MEMORY_POOL.with_borrow_mut(|memory_pool| memory_pool.get_call_frames()); + let mut execution_mode = + if use_jit { ExecutionMode::PreferJit } else { ExecutionMode::Interpreted }; + let mut call_frames = std::iter::repeat_with(|| CallFrame { + caller_saved_registers: [0; ebpf::SCRATCH_REGS], + frame_pointer: 0, + target_pc: 0, + }) + .take(executable.get_config().max_call_depth) + .collect::>(); let (compute_units_consumed, result) = vm.execute_program(executable, &mut execution_mode, &mut call_frames); let register_trace = std::mem::take(&mut vm.register_trace); MEMORY_POOL.with_borrow_mut(|memory_pool| { memory_pool.put_stack(stack); memory_pool.put_heap(heap); - memory_pool.put_call_frames(call_frames); - debug_assert!(memory_pool.stack_len() <= MAX_INSTRUCTION_STACK_DEPTH_SIMD_0268); - debug_assert!(memory_pool.heap_len() <= MAX_INSTRUCTION_STACK_DEPTH_SIMD_0268); + debug_assert!(memory_pool.stack_len() <= MAX_INSTRUCTION_STACK_DEPTH); + debug_assert!(memory_pool.heap_len() <= MAX_INSTRUCTION_STACK_DEPTH); }); drop(vm); invoke_context.insert_register_trace(register_trace); - - // This section is a little convoluted due to the nested and sibling (CPI) invocations. - let total_execute_ns = execute_time.end_as_ns(); - let nested_execution_time_delta = invoke_context - .total_nested_exec_time - .saturating_sub(prev_nested_exec_time); - let this_call_ns = - total_execute_ns.saturating_sub(nested_execution_time_delta.as_nanos() as u64); - invoke_context.total_nested_exec_time = invoke_context - .total_nested_exec_time - .saturating_add(Duration::from_nanos(this_call_ns)); - let this_call_us = this_call_ns / 1000; - invoke_context.timings.execute_us += this_call_us; - match execution_mode { - ExecutionMode::Interpreted => cache_entry.stats.interpreter_executed(this_call_us), - ExecutionMode::Jit => cache_entry.stats.jit_executed(this_call_us), - ExecutionMode::PreferJit => { /* not actually executed? */ } - } + execute_time.stop(); + invoke_context.timings.execute_us += execute_time.as_us(); ic_logger_msg!( log_collector, @@ -360,9 +279,7 @@ pub fn execute<'a, 'b: 'a>( // Don't clean me up!! // This feature is active on all networks, but we still toggle // it off during fuzzing. - if invoke_context - .get_feature_set() - .deplete_cu_meter_on_vm_failure + if invoke_context.get_feature_set().deplete_cu_meter_on_vm_failure && !matches!(error, EbpfError::SyscallError(_)) { // when an exception is thrown during the execution of a @@ -378,40 +295,38 @@ pub fn execute<'a, 'b: 'a>( invoke_context.consume(invoke_context.get_remaining()); } - if virtual_address_space_adjustments { - if let EbpfError::SyscallError(err) = error { - error = err - .downcast::() - .map(|err| *err) - .unwrap_or_else(EbpfError::SyscallError); - } - if let EbpfError::AccessViolation(access_type, vm_addr, len, _section_name) = - error + if let EbpfError::SyscallError(err) = error { + error = err + .downcast::() + .map(|err| *err) + .unwrap_or_else(EbpfError::SyscallError); + } + if let EbpfError::AccessViolation(access_type, vm_addr, len, _section_name) = error + { + // Account data is directly mapped. A write to readonly data + // or an access into the reserved growth range appears as a + // memory violation; map it to the account-specific error. + if let Some((instruction_account_index, vm_addr_range)) = account_region_addrs + .iter() + .enumerate() + .find(|(_, vm_addr_range)| vm_addr_range.contains(&vm_addr)) { - // If virtual_address_space_adjustments is enabled and a program tries to write to a readonly - // region we'll get a memory access violation. Map it to a more specific - // error so it's easier for developers to see what happened. - if let Some((instruction_account_index, vm_addr_range)) = - account_region_addrs - .iter() - .enumerate() - .find(|(_, vm_addr_range)| vm_addr_range.contains(&vm_addr)) - { - let transaction_context = &invoke_context.transaction_context; - let instruction_context = - transaction_context.get_current_instruction_context()?; - let account = instruction_context.try_borrow_instruction_account( - instruction_account_index as IndexOfAccount, - )?; - if vm_addr.saturating_add(len) <= vm_addr_range.end { - // The access was within the range of the accounts address space, - // but it might not be within the range of the actual data. - let is_access_outside_of_data = vm_addr - .saturating_add(len) - .saturating_sub(vm_addr_range.start) + let transaction_context = &invoke_context.transaction_context; + let instruction_context = + transaction_context.get_current_instruction_context()?; + let account = instruction_context.try_borrow_instruction_account( + instruction_account_index as IndexOfAccount, + )?; + if vm_addr.saturating_add(len) <= vm_addr_range.end { + // The access was within the range of the account address space, + // but it might not be within the range of the actual data. + let is_access_outside_of_data = + vm_addr.saturating_add(len).saturating_sub(vm_addr_range.start) as usize > account.get_data().len(); - error = EbpfError::SyscallError(Box::new(match access_type { + error = EbpfError::SyscallError(Box::new( + #[allow(deprecated)] + match access_type { AccessType::Store => { if let Err(err) = account.can_data_be_changed() { err @@ -434,16 +349,12 @@ pub fn execute<'a, 'b: 'a>( InstructionError::InvalidRealloc } } - })); - } + }, + )); } } } - Err(if let EbpfError::SyscallError(err) = error { - err - } else { - error.into() - }) + Err(if let EbpfError::SyscallError(err) = error { err } else { error.into() }) } _ => Ok(()), } @@ -452,32 +363,18 @@ pub fn execute<'a, 'b: 'a>( fn deserialize_parameters( invoke_context: &mut InvokeContext, parameter_bytes: &[u8], - virtual_address_space_adjustments: bool, - account_data_direct_mapping: bool, ) -> Result<(), InstructionError> { serialization::deserialize_parameters( - &invoke_context - .transaction_context - .get_current_instruction_context()?, - virtual_address_space_adjustments, - account_data_direct_mapping, + &invoke_context.transaction_context.get_current_instruction_context()?, parameter_bytes, - &invoke_context - .memory_contexts - .memory_context_abi_v1()? - .accounts_metadata, + &invoke_context.get_syscall_context()?.accounts_metadata, ) } let mut deserialize_time = Measure::start("deserialize"); let execute_or_deserialize_result = execution_result.and_then(|_| { - deserialize_parameters( - invoke_context, - parameter_bytes.as_slice(), - virtual_address_space_adjustments, - account_data_direct_mapping, - ) - .map_err(|error| Box::new(error) as Box) + deserialize_parameters(invoke_context, parameter_bytes.as_slice()) + .map_err(|error| Box::new(error) as Box) }); deserialize_time.stop();