From 47b19b5841338d1db62583f3fdad8229bbe1caa0 Mon Sep 17 00:00:00 2001 From: Kevin Heavey Date: Tue, 12 Mar 2024 01:05:13 +0400 Subject: [PATCH] update the program cache whenever a program account is updated --- Cargo.lock | 36 ++++--- Cargo.toml | 3 +- benches/banks_client_comparison.rs | 2 +- src/accounts_db.rs | 126 +++++++++++++++++++++-- src/bank.rs | 155 ++++++----------------------- tests/counter_test.rs | 6 +- tests/loaders.rs | 2 - tests/spl.rs | 2 +- tests/system.rs | 6 +- 9 files changed, 177 insertions(+), 161 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f4df02..d84c330 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,7 +182,7 @@ dependencies = [ "ark-std", "derivative", "hashbrown 0.13.2", - "itertools", + "itertools 0.10.5", "num-traits", "zeroize", ] @@ -199,7 +199,7 @@ dependencies = [ "ark-std", "derivative", "digest 0.10.7", - "itertools", + "itertools 0.10.5", "num-bigint 0.4.4", "num-traits", "paste", @@ -968,7 +968,7 @@ dependencies = [ "clap 4.5.0", "criterion-plot", "is-terminal", - "itertools", + "itertools 0.10.5", "num-traits", "once_cell", "oorandom", @@ -989,7 +989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -1939,6 +1939,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -2071,6 +2080,7 @@ version = "0.1.0" dependencies = [ "bincode", "criterion", + "itertools 0.12.1", "solana-bpf-loader-program", "solana-compute-budget-program", "solana-loader-v4-program", @@ -3527,7 +3537,7 @@ dependencies = [ "fs-err", "im", "index_list", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "lz4", @@ -3958,7 +3968,7 @@ dependencies = [ "console_log", "curve25519-dalek", "getrandom 0.2.12", - "itertools", + "itertools 0.10.5", "js-sys", "lazy_static", "libc", @@ -3998,7 +4008,7 @@ dependencies = [ "bincode", "eager", "enum-iterator", - "itertools", + "itertools 0.10.5", "libc", "log", "num-derive 0.3.3", @@ -4080,7 +4090,7 @@ dependencies = [ "async-mutex", "async-trait", "futures", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "quinn", @@ -4210,7 +4220,7 @@ dependencies = [ "fs-err", "im", "index_list", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "lru", @@ -4286,7 +4296,7 @@ dependencies = [ "ed25519-dalek-bip32", "generic-array", "hmac 0.12.1", - "itertools", + "itertools 0.10.5", "js-sys", "lazy_static", "libsecp256k1", @@ -4381,7 +4391,7 @@ dependencies = [ "futures-util", "histogram", "indexmap 2.2.2", - "itertools", + "itertools 0.10.5", "libc", "log", "nix", @@ -4517,7 +4527,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2072f8e6660f12372124a7988b91aa5402c26044ba8f87f59f90ad227e358002" dependencies = [ "crossbeam-channel", - "itertools", + "itertools 0.10.5", "log", "rustc_version", "serde", @@ -4578,7 +4588,7 @@ dependencies = [ "byteorder", "curve25519-dalek", "getrandom 0.1.16", - "itertools", + "itertools 0.10.5", "lazy_static", "merlin", "num-derive 0.3.3", diff --git a/Cargo.toml b/Cargo.toml index b50068a..e75a902 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,9 @@ version = "0.1.0" edition = "2021" [dependencies] +itertools = "0.12" thiserror = "1.0" +solana-program = "=1.17.18" solana-program-runtime = "=1.17.18" solana-bpf-loader-program = "=1.17.18" solana-sdk = "=1.17.18" @@ -16,7 +18,6 @@ bincode = "1.3" [dev-dependencies] spl-token = "3.5.0" -solana-program = "=1.17.18" solana-program-test = "=1.17.18" criterion = "0.5" tokio = "1.35" diff --git a/benches/banks_client_comparison.rs b/benches/banks_client_comparison.rs index 88f2f73..0c0826d 100644 --- a/benches/banks_client_comparison.rs +++ b/benches/banks_client_comparison.rs @@ -111,7 +111,7 @@ fn criterion_benchmark(c: &mut Criterion) { ); svm.send_transaction(tx.clone()).unwrap(); } - assert_eq!(svm.get_account(&counter_address).data[0], NUM_GREETINGS); + assert_eq!(svm.get_account(&counter_address).unwrap().data[0], NUM_GREETINGS); }) }); group.bench_function("banks_client_bench", |b| { diff --git a/src/accounts_db.rs b/src/accounts_db.rs index f747d94..936065f 100644 --- a/src/accounts_db.rs +++ b/src/accounts_db.rs @@ -1,26 +1,132 @@ -use solana_sdk::{account::AccountSharedData, pubkey::Pubkey}; -use std::collections::HashMap; +use solana_program::{ + bpf_loader, bpf_loader_deprecated, + bpf_loader_upgradeable::{self, UpgradeableLoaderState}, + clock::Clock, + instruction::InstructionError, + loader_v4::{self, LoaderV4State}, + sysvar, +}; +use solana_program_runtime::loaded_programs::{ + LoadProgramMetrics, LoadedProgram, LoadedProgramsForTxBatch, +}; +use solana_sdk::{ + account::{AccountSharedData, ReadableAccount}, + account_utils::StateMut, + pubkey::Pubkey, +}; +use std::{collections::HashMap, sync::Arc}; #[derive(Default)] pub(crate) struct AccountsDb { inner: HashMap, + pub(crate) programs_cache: LoadedProgramsForTxBatch, } impl AccountsDb { - pub(crate) fn get_account(&self, pubkey: &Pubkey) -> AccountSharedData { - self.inner - .get(pubkey) - .map(|acc| acc.to_owned()) - .unwrap_or_default() + pub(crate) fn get_account(&self, pubkey: &Pubkey) -> Option { + self.inner.get(pubkey).map(|acc| acc.to_owned()) } pub(crate) fn add_account(&mut self, pubkey: Pubkey, data: AccountSharedData) { + if data.executable() && pubkey != Pubkey::default() { + let loaded_program = self.load_program(&data).unwrap(); + self.programs_cache + .replenish(pubkey, Arc::new(loaded_program)); + } self.inner.insert(pubkey, data); } - pub(crate) fn sync_accounts(&mut self, accounts: Vec<(Pubkey, AccountSharedData)>) { - for (pubkey, data) in accounts { - self.add_account(pubkey, data); + /// Skip the executable() checks for builtin accounts + pub(crate) fn add_builtin_account(&mut self, pubkey: Pubkey, data: AccountSharedData) { + self.inner.insert(pubkey, data); + } + + pub(crate) fn sync_accounts(&mut self, mut accounts: Vec<(Pubkey, AccountSharedData)>) { + // need to add programdata accounts first if there are any + itertools::partition(&mut accounts, |x| { + x.1.owner() == &bpf_loader_upgradeable::id() + && x.1.data().get(0).map_or(false, |byte| *byte == 3) + }); + for (pubkey, acc) in accounts { + self.add_account(pubkey, acc); + } + } + + fn load_program( + &self, + program_account: &AccountSharedData, + // programdata_account: Option<&AccountSharedData> + ) -> Result { + let metrics = &mut LoadProgramMetrics::default(); + + let owner = program_account.owner(); + let program_runtime_v1 = self.programs_cache.environments.program_runtime_v1.clone(); + let clock_acc = self.get_account(&sysvar::clock::ID); + let clock: Clock = clock_acc + .map(|x| bincode::deserialize::(x.data()).unwrap()) + .unwrap_or_default(); + let slot = clock.slot; + + if bpf_loader::check_id(owner) | bpf_loader_deprecated::check_id(owner) { + LoadedProgram::new( + owner, + self.programs_cache.environments.program_runtime_v1.clone(), + slot, + slot, + None, + program_account.data(), + program_account.data().len(), + &mut LoadProgramMetrics::default(), + ) + .map_err(|_| InstructionError::InvalidAccountData) + } else if bpf_loader_upgradeable::check_id(owner) { + let Ok(UpgradeableLoaderState::Program { + programdata_address, + }) = program_account.state() + else { + return Err(InstructionError::InvalidAccountData); + }; + let programdata_account = self.get_account(&programdata_address).unwrap(); + let program_data = programdata_account.data(); + program_data + .get(UpgradeableLoaderState::size_of_programdata_metadata()..) + .ok_or(Box::new(InstructionError::InvalidAccountData).into()) + .and_then(|programdata| { + LoadedProgram::new( + owner, + program_runtime_v1, + slot, + slot, + None, + programdata, + program_account + .data() + .len() + .saturating_add(program_data.len()), + metrics, + ) + }) + .map_err(|_| InstructionError::InvalidAccountData) + } else if loader_v4::check_id(owner) { + program_account + .data() + .get(LoaderV4State::program_data_offset()..) + .ok_or(Box::new(InstructionError::InvalidAccountData).into()) + .and_then(|elf_bytes| { + LoadedProgram::new( + &loader_v4::id(), + program_runtime_v1, + slot, + slot, + None, + elf_bytes, + program_account.data().len(), + metrics, + ) + }) + .map_err(|_| InstructionError::InvalidAccountData) + } else { + Err(InstructionError::IncorrectProgramId) } } } diff --git a/src/bank.rs b/src/bank.rs index d3ec8f1..2e725ad 100644 --- a/src/bank.rs +++ b/src/bank.rs @@ -11,14 +11,10 @@ use solana_program_runtime::{ }; use solana_sdk::{ account::{Account, AccountSharedData, ReadableAccount, WritableAccount}, - account_utils::StateMut, - bpf_loader, bpf_loader_deprecated, - bpf_loader_upgradeable::{self, UpgradeableLoaderState}, + bpf_loader, clock::Clock, feature_set::FeatureSet, hash::Hash, - instruction::InstructionError, - loader_v4::{self, LoaderV4State}, message::{ v0::{LoadedAddresses, MessageAddressTableLookup}, AddressLoader, AddressLoaderError, Message, VersionedMessage, @@ -46,7 +42,6 @@ use crate::{ spl::load_spl_programs, types::{ExecutionResult, FailedTransactionMetadata, TransactionMetadata, TransactionResult}, utils::RentState, - PROGRAM_OWNERS, }; #[derive(Clone, Default)] @@ -64,7 +59,6 @@ impl AddressLoader for LightAddressLoader { pub struct LiteSVM { accounts: AccountsDb, //TODO compute budget - programs_cache: LoadedProgramsForTxBatch, airdrop_kp: Keypair, sysvar_cache: SysvarCache, feature_set: Arc, @@ -79,7 +73,6 @@ impl Default for LiteSVM { fn default() -> Self { Self { accounts: Default::default(), - programs_cache: Default::default(), airdrop_kp: Keypair::new(), sysvar_cache: Default::default(), feature_set: Default::default(), @@ -113,9 +106,9 @@ impl LiteSVM { BUILTINS.iter().for_each(|builtint| { let loaded_program = LoadedProgram::new_builtin(0, builtint.name.len(), builtint.entrypoint); - self.programs_cache + self.accounts.programs_cache .replenish(builtint.program_id, Arc::new(loaded_program)); - self.accounts.add_account( + self.accounts.add_builtin_account( builtint.program_id, native_loader::create_loadable_account_for_test(builtint.name), ); @@ -136,8 +129,8 @@ impl LiteSVM { let program_runtime_v2 = create_program_runtime_environment_v2(&ComputeBudget::default(), true); - self.programs_cache.environments.program_runtime_v1 = Arc::new(program_runtime_v1); - self.programs_cache.environments.program_runtime_v2 = Arc::new(program_runtime_v2); + self.accounts.programs_cache.environments.program_runtime_v1 = Arc::new(program_runtime_v1); + self.accounts.programs_cache.environments.program_runtime_v2 = Arc::new(program_runtime_v2); self.feature_set = Arc::new(feature_set); self } @@ -164,16 +157,16 @@ impl LiteSVM { ) } - pub fn get_account(&self, pubkey: &Pubkey) -> Account { - self.accounts.get_account(pubkey).into() + pub fn get_account(&self, pubkey: &Pubkey) -> Option { + self.accounts.get_account(pubkey).map(Into::into) } pub fn set_account(&mut self, pubkey: Pubkey, data: Account) { self.accounts.add_account(pubkey, data.into()) } - pub fn get_balance(&self, pubkey: &Pubkey) -> u64 { - self.accounts.get_account(pubkey).lamports() + pub fn get_balance(&self, pubkey: &Pubkey) -> Option { + self.accounts.get_account(pubkey).map(|x| x.lamports()) } pub fn latest_blockhash(&self) -> Hash { @@ -231,7 +224,7 @@ impl LiteSVM { pub fn add_builtin(&mut self, program_id: Pubkey, entrypoint: BuiltinFunctionWithContext) { let builtin = LoadedProgram::new_builtin(self.slot, 1, entrypoint); - self.programs_cache.replenish(program_id, Arc::new(builtin)); + self.accounts.programs_cache.replenish(program_id, Arc::new(builtin)); self.accounts .add_account(program_id, AccountSharedData::new(0, 1, &bpf_loader::id())); } @@ -251,94 +244,15 @@ impl LiteSVM { account.owner(), account.data().len(), self.slot, - self.programs_cache.environments.program_runtime_v1.clone(), + self.accounts.programs_cache.environments.program_runtime_v1.clone(), false, ) .unwrap_or_default(); self.accounts.add_account(program_id, account); - self.programs_cache + self.accounts.programs_cache .replenish(program_id, Arc::new(loaded_program)); } - //TODO handle reload - pub(crate) fn load_program( - &self, - program_id: &Pubkey, - ) -> Result { - let program_account = self.accounts.get_account(program_id); - let metrics = &mut LoadProgramMetrics::default(); - - if !program_account.executable() { - return Err(InstructionError::AccountNotExecutable); - }; - - let owner = program_account.owner(); - let program_runtime_v1 = self.programs_cache.environments.program_runtime_v1.clone(); - - if bpf_loader::check_id(owner) | bpf_loader_deprecated::check_id(owner) { - LoadedProgram::new( - owner, - self.programs_cache.environments.program_runtime_v1.clone(), - self.slot, - self.slot, - None, - program_account.data(), - program_account.data().len(), - &mut LoadProgramMetrics::default(), - ) - .map_err(|_| InstructionError::InvalidAccountData) - } else if bpf_loader_upgradeable::check_id(owner) { - let Ok(UpgradeableLoaderState::Program { - programdata_address, - }) = program_account.state() - else { - return Err(InstructionError::InvalidAccountData); - }; - let programdata_account = self.accounts.get_account(&programdata_address); - - programdata_account - .data() - .get(UpgradeableLoaderState::size_of_programdata_metadata()..) - .ok_or(Box::new(InstructionError::InvalidAccountData).into()) - .and_then(|programdata| { - LoadedProgram::new( - owner, - program_runtime_v1, - self.slot, - self.slot, - None, - programdata, - program_account - .data() - .len() - .saturating_add(programdata_account.data().len()), - metrics, - ) - }) - .map_err(|_| InstructionError::InvalidAccountData) - } else if loader_v4::check_id(owner) { - program_account - .data() - .get(LoaderV4State::program_data_offset()..) - .ok_or(Box::new(InstructionError::InvalidAccountData).into()) - .and_then(|elf_bytes| { - LoadedProgram::new( - &loader_v4::id(), - program_runtime_v1, - self.slot, - self.slot, - None, - elf_bytes, - program_account.data().len(), - metrics, - ) - }) - .map_err(|_| InstructionError::InvalidAccountData) - } else { - Err(InstructionError::IncorrectProgramId) - } - } - //TODO fn create_transaction_context( &mut self, @@ -349,7 +263,7 @@ impl LiteSVM { .message() .account_keys() .iter() - .map(|p| (*p, self.accounts.get_account(p))) + .map(|p| (*p, self.accounts.get_account(p).unwrap_or_default())) .collect(); TransactionContext::new( @@ -388,32 +302,16 @@ impl LiteSVM { //reload program cache let mut programs_modified_by_tx = - LoadedProgramsForTxBatch::new(self.slot, self.programs_cache.environments.clone()); + LoadedProgramsForTxBatch::new(self.slot, self.accounts.programs_cache.environments.clone()); let mut programs_updated_only_for_global_cache = LoadedProgramsForTxBatch::default(); let mut accumulated_consume_units = 0; - let Ok(program_indices) = tx + let program_indices = tx .message() .instructions() .iter() - .map(|c| { - let program_id = context.get_key_of_account_at_index(c.program_id_index.into())?; - - if !PROGRAM_OWNERS.contains(program_id) { - let loaded_program = self.load_program(program_id)?; - self.programs_cache - .replenish(*program_id, Arc::new(loaded_program)); - } - - Ok(vec![c.program_id_index as u16]) - }) - .collect::>, InstructionError>>() - else { - return ( - Err(TransactionError::InvalidProgramForExecution), - accumulated_consume_units, - ); - }; + .map(|c| vec![c.program_id_index as u16]) + .collect::>>(); let mut tx_result = MessageProcessor::process_message( tx.message(), @@ -421,7 +319,7 @@ impl LiteSVM { context, *self.sysvar_cache.get_rent().unwrap_or_default(), Some(self.log_collector.clone()), - &self.programs_cache, + &self.accounts.programs_cache, &mut programs_modified_by_tx, &mut programs_updated_only_for_global_cache, self.feature_set.clone(), @@ -461,7 +359,7 @@ impl LiteSVM { if !account.data().is_empty() { let post_rent_state = RentState::from_account(&account, &rent); let pre_rent_state = - RentState::from_account(&self.accounts.get_account(pubkey), &rent); + RentState::from_account(&self.accounts.get_account(pubkey).unwrap_or_default(), &rent); if !post_rent_state.transition_allowed_from(&pre_rent_state) { return Err(TransactionError::InsufficientFundsForRent { @@ -494,22 +392,26 @@ impl LiteSVM { } let mut context = self.create_transaction_context(&sanitized_tx, compute_budget); - let (result, compute_units_consumed) = self.process_transaction(&sanitized_tx, compute_budget, &mut context); let signature = sanitized_tx.signature().to_owned(); - let ExecutionRecord { accounts, return_data, touched_account_count: _, accounts_resize_delta: _, } = context.into(); + let msg = sanitized_tx.message(); + let post_accounts = accounts + .into_iter() + .enumerate() + .filter_map(|(idx, pair)| msg.is_writable(idx).then_some(pair)) + .collect(); ExecutionResult { tx_result: result, signature, - post_accounts: accounts, + post_accounts, compute_units_consumed, return_data, } @@ -525,13 +427,14 @@ impl LiteSVM { } pub fn send_transaction(&mut self, tx: impl Into) -> TransactionResult { + let vtx: VersionedTransaction = tx.into(); let ExecutionResult { post_accounts, tx_result, signature, compute_units_consumed, return_data, - } = self.execute_transaction(tx.into()); + } = self.execute_transaction(vtx); let meta = TransactionMetadata { logs: self.log_collector.take().into_messages(), @@ -583,6 +486,6 @@ impl LiteSVM { self.expire_blockhash(); self.slot = slot; self.block_height = slot; - self.programs_cache.set_slot_for_tests(slot); + self.accounts.programs_cache.set_slot_for_tests(slot); } } diff --git a/tests/counter_test.rs b/tests/counter_test.rs index 4e14e6c..5ea973f 100644 --- a/tests/counter_test.rs +++ b/tests/counter_test.rs @@ -24,7 +24,6 @@ pub fn integration_test() { let payer_pk = payer_kp.pubkey(); let program_id = Pubkey::new_unique(); svm.store_program(program_id, &read_counter_program()); - svm.airdrop(&payer_pk, 1000000000).unwrap(); let blockhash = svm.latest_blockhash(); let counter_address = Pubkey::new_unique(); @@ -38,7 +37,7 @@ pub fn integration_test() { }, ); assert_eq!( - svm.get_account(&counter_address).data, + svm.get_account(&counter_address).unwrap().data, 0u32.to_le_bytes().to_vec() ); let num_greets = 100u8; @@ -54,7 +53,7 @@ pub fn integration_test() { svm.send_transaction(tx).unwrap(); } assert_eq!( - svm.get_account(&counter_address).data, + svm.get_account(&counter_address).unwrap().data, (num_greets as u32).to_le_bytes().to_vec() ); } @@ -137,7 +136,6 @@ async fn do_program_test(program_id: Pubkey, counter_address: Pubkey) { .await .unwrap(); tx_res.result.unwrap(); - println!("logs: {:?}", tx_res.metadata.unwrap().log_messages); } let fetched = ctx .banks_client diff --git a/tests/loaders.rs b/tests/loaders.rs index 7fc75c1..604a422 100644 --- a/tests/loaders.rs +++ b/tests/loaders.rs @@ -78,13 +78,11 @@ fn hello_world_with_deploy_upgradeable() { let program_id = bank .deploy_upgradeable_program(&payer_kp, program_bytes) .unwrap(); - let instruction = Instruction::new_with_bytes(program_id, &[], vec![AccountMeta::new(payer_pk, true)]); let message = Message::new(&[instruction], Some(&payer_pk)); let tx = Transaction::new(&[&payer_kp], message, bank.latest_blockhash()); let tx_result = bank.send_transaction(tx); - assert!(tx_result.is_ok()); assert!(tx_result .unwrap() diff --git a/tests/spl.rs b/tests/spl.rs index dbe605a..677cb65 100644 --- a/tests/spl.rs +++ b/tests/spl.rs @@ -37,7 +37,7 @@ fn spl_token() { assert!(tx_result.is_ok()); let mint_acc = svm.get_account(&mint_kp.pubkey()); - let mint = spl_token::state::Mint::unpack(&mint_acc.data).unwrap(); + let mint = spl_token::state::Mint::unpack(&mint_acc.unwrap().data).unwrap(); assert_eq!(mint.decimals, 8); assert_eq!(mint.mint_authority, Some(payer_pk).into()); diff --git a/tests/system.rs b/tests/system.rs index 6286d76..32048d1 100644 --- a/tests/system.rs +++ b/tests/system.rs @@ -28,8 +28,8 @@ fn system_transfer() { let to_account = bank.get_account(&to); assert!(tx_res.is_ok()); - assert_eq!(from_account.lamports, 36); - assert_eq!(to_account.lamports, 64); + assert_eq!(from_account.unwrap().lamports, 36); + assert_eq!(to_account.unwrap().lamports, 64); } #[test] @@ -57,7 +57,7 @@ fn system_create_account() { ); let tx_res = bank.send_transaction(tx); - let account = bank.get_account(&new_account.pubkey()); + let account = bank.get_account(&new_account.pubkey()).unwrap(); assert!(tx_res.is_ok()); assert_eq!(account.lamports, lamports);