From f5860e53a338b360a6dfb233469f6e57bc178656 Mon Sep 17 00:00:00 2001 From: George Mitenkov Date: Sat, 3 Aug 2024 10:44:17 +0100 Subject: [PATCH 01/87] [move] Full loader integration on MoveVM side (#14075) --- .../aptos-release-builder/src/simulate.rs | 6 +- .../aptos-vm-profiling/src/bins/run_move.rs | 6 +- aptos-move/aptos-vm/src/aptos_vm.rs | 53 +- .../aptos-vm/src/move_vm_ext/session/mod.rs | 2 + .../aptos-vm/src/move_vm_ext/warm_vm_cache.rs | 1 + .../aptos-vm/src/transaction_validation.rs | 9 +- aptos-move/aptos-vm/src/validator_txns/dkg.rs | 6 +- aptos-move/aptos-vm/src/validator_txns/jwk.rs | 6 +- .../aptos-vm/src/verifier/event_validation.rs | 17 +- .../aptos-vm/src/verifier/randomness.rs | 1 + .../aptos-vm/src/verifier/resource_groups.rs | 15 +- .../verifier/transaction_arg_validation.rs | 14 +- aptos-move/e2e-tests/src/executor.rs | 17 +- aptos-move/framework/src/module_metadata.rs | 4 + aptos-move/vm-genesis/src/lib.rs | 8 +- .../async/move-async-vm/src/async_vm.rs | 455 +++++++++ .../async/move-async-vm/tests/testsuite.rs | 429 +++++++++ .../src/tests/bad_entry_point_tests.rs | 4 +- .../src/tests/bad_storage_tests.rs | 21 +- .../src/tests/binary_format_version.rs | 16 +- .../src/tests/exec_func_effects_tests.rs | 5 +- .../src/tests/function_arg_tests.rs | 3 +- .../src/tests/instantiation_tests.rs | 6 +- .../src/tests/invariant_violation_tests.rs | 4 +- .../integration-tests/src/tests/leak_tests.rs | 4 +- .../src/tests/loader_tests.rs | 22 +- .../src/tests/mutated_accounts_tests.rs | 14 +- .../src/tests/native_tests.rs | 26 +- .../src/tests/nested_loop_tests.rs | 17 +- .../src/tests/regression_tests.rs | 4 +- .../src/tests/return_value_tests.rs | 5 +- .../tests/runtime_reentrancy_check_tests.rs | 11 +- .../src/tests/vm_arguments_tests.rs | 7 +- .../move/move-vm/runtime/src/data_cache.rs | 74 +- .../move/move-vm/runtime/src/interpreter.rs | 304 +++--- third_party/move/move-vm/runtime/src/lib.rs | 4 + .../move-vm/runtime/src/loader/function.rs | 6 +- .../move/move-vm/runtime/src/loader/mod.rs | 903 ++++++++++++------ .../move-vm/runtime/src/loader/modules.rs | 39 +- .../move/move-vm/runtime/src/loader/script.rs | 46 +- .../move/move-vm/runtime/src/move_vm.rs | 49 +- .../move-vm/runtime/src/native_functions.rs | 54 +- .../move/move-vm/runtime/src/runtime.rs | 52 +- .../move/move-vm/runtime/src/session.rs | 106 +- .../move/move-vm/runtime/src/storage/dummy.rs | 124 +++ .../move-vm/runtime/src/storage/loader.rs | 375 ++++++++ .../move/move-vm/runtime/src/storage/mod.rs | 15 + .../runtime/src/storage/module_storage.rs | 56 ++ .../runtime/src/storage/script_storage.rs | 37 + .../src/storage/struct_name_index_map.rs | 97 ++ .../src/storage/struct_type_storage.rs | 57 ++ .../move-vm/runtime/src/storage/verifier.rs | 30 + .../testing-infra/test-generation/src/lib.rs | 3 +- .../src/vm_test_harness.rs | 7 +- .../tools/move-unit-test/src/test_runner.rs | 2 + 55 files changed, 2966 insertions(+), 692 deletions(-) create mode 100644 third_party/move/extensions/async/move-async-vm/src/async_vm.rs create mode 100644 third_party/move/extensions/async/move-async-vm/tests/testsuite.rs create mode 100644 third_party/move/move-vm/runtime/src/storage/dummy.rs create mode 100644 third_party/move/move-vm/runtime/src/storage/loader.rs create mode 100644 third_party/move/move-vm/runtime/src/storage/mod.rs create mode 100644 third_party/move/move-vm/runtime/src/storage/module_storage.rs create mode 100644 third_party/move/move-vm/runtime/src/storage/script_storage.rs create mode 100644 third_party/move/move-vm/runtime/src/storage/struct_name_index_map.rs create mode 100644 third_party/move/move-vm/runtime/src/storage/struct_type_storage.rs create mode 100644 third_party/move/move-vm/runtime/src/storage/verifier.rs diff --git a/aptos-move/aptos-release-builder/src/simulate.rs b/aptos-move/aptos-release-builder/src/simulate.rs index 0909e62d6df8d..202c27617e9ba 100644 --- a/aptos-move/aptos-release-builder/src/simulate.rs +++ b/aptos-move/aptos-release-builder/src/simulate.rs @@ -66,7 +66,10 @@ use move_core_types::{ language_storage::{ModuleId, StructTag}, move_resource::MoveResource, }; -use move_vm_runtime::module_traversal::{TraversalContext, TraversalStorage}; +use move_vm_runtime::{ + module_traversal::{TraversalContext, TraversalStorage}, + DummyCodeStorage, +}; use move_vm_types::{gas::UnmeteredGasMeter, resolver::ModuleResolver}; use once_cell::sync::Lazy; use parking_lot::Mutex; @@ -480,6 +483,7 @@ fn force_end_epoch(state_view: &SimulationStateView) -> Result<( vec![bcs::to_bytes(&AccountAddress::ONE)?], &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, )?; let (mut change_set, module_write_set) = sess.finish(&change_set_configs)?; change_set.try_materialize_aggregator_v1_delta_set(&resolver)?; diff --git a/aptos-move/aptos-vm-profiling/src/bins/run_move.rs b/aptos-move/aptos-vm-profiling/src/bins/run_move.rs index 9358930eac58e..8dcdb890c9040 100644 --- a/aptos-move/aptos-vm-profiling/src/bins/run_move.rs +++ b/aptos-move/aptos-vm-profiling/src/bins/run_move.rs @@ -12,7 +12,7 @@ use move_core_types::{account_address::AccountAddress, ident_str, identifier::Id use move_ir_compiler::Compiler; use move_vm_runtime::{ module_traversal::*, move_vm::MoveVM, native_extensions::NativeContextExtensions, - native_functions::NativeFunction, + native_functions::NativeFunction, DummyCodeStorage, }; use move_vm_test_utils::InMemoryStorage; use move_vm_types::{ @@ -176,6 +176,8 @@ fn main() -> Result<()> { args, &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, )?; } else { let module = Compiler::new(test_modules.iter().collect()).into_compiled_module(&src)?; @@ -186,6 +188,7 @@ fn main() -> Result<()> { module_blob, *module.self_id().address(), &mut UnmeteredGasMeter, + &DummyCodeStorage, )?; let args: Vec> = vec![]; let res = sess.execute_function_bypass_visibility( @@ -195,6 +198,7 @@ fn main() -> Result<()> { args, &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, )?; println!("{:?}", res); } diff --git a/aptos-move/aptos-vm/src/aptos_vm.rs b/aptos-move/aptos-vm/src/aptos_vm.rs index 125146a15ba8b..fd822f4e19cee 100644 --- a/aptos-move/aptos-vm/src/aptos_vm.rs +++ b/aptos-move/aptos-vm/src/aptos_vm.rs @@ -106,6 +106,7 @@ use move_core_types::{ use move_vm_runtime::{ logging::expect_no_verification_errors, module_traversal::{TraversalContext, TraversalStorage}, + DummyCodeStorage, }; use move_vm_types::gas::{GasMeter, UnmeteredGasMeter}; use num_cpus; @@ -733,13 +734,20 @@ impl AptosVM { // the error semantics. if self.gas_feature_version >= 15 { session.check_script_dependencies_and_check_gas( + &DummyCodeStorage, + &DummyCodeStorage, gas_meter, traversal_context, script.code(), )?; } - let func = session.load_script(script.code(), script.ty_args())?; + let func = session.load_script( + &DummyCodeStorage, + &DummyCodeStorage, + script.code(), + script.ty_args(), + )?; let compiled_script = match CompiledScript::deserialize_with_config( script.code(), @@ -780,6 +788,8 @@ impl AptosVM { args, gas_meter, traversal_context, + &DummyCodeStorage, + &DummyCodeStorage, )?; Ok(()) } @@ -801,14 +811,20 @@ impl AptosVM { let module_id = traversal_context .referenced_module_ids .alloc(entry_fn.module().clone()); - session.check_dependencies_and_charge_gas(gas_meter, traversal_context, [( - module_id.address(), - module_id.name(), - )])?; + session.check_dependencies_and_charge_gas( + &DummyCodeStorage, + gas_meter, + traversal_context, + [(module_id.address(), module_id.name())], + )?; } - let function = - session.load_function(entry_fn.module(), entry_fn.function(), entry_fn.ty_args())?; + let function = session.load_function( + &DummyCodeStorage, + entry_fn.module(), + entry_fn.function(), + entry_fn.ty_args(), + )?; // Native entry function is forbidden. if self @@ -845,7 +861,13 @@ impl AptosVM { &function, struct_constructors_enabled, )?; - session.execute_entry_function(function, args, gas_meter, traversal_context)?; + session.execute_entry_function( + function, + args, + gas_meter, + traversal_context, + &DummyCodeStorage, + )?; Ok(()) } @@ -1105,6 +1127,7 @@ impl AptosVM { ]), gas_meter, traversal_context, + &DummyCodeStorage, ) })? .return_values @@ -1203,6 +1226,7 @@ impl AptosVM { cleanup_args, &mut UnmeteredGasMeter, traversal_context, + &DummyCodeStorage, ) .map_err(|e| e.into_vm_status()) })?; @@ -1341,6 +1365,7 @@ impl AptosVM { cleanup_args, &mut UnmeteredGasMeter, traversal_context, + &DummyCodeStorage, ) .map_err(|e| e.into_vm_status()) })?; @@ -1365,7 +1390,8 @@ impl AptosVM { continue; } *new_published_modules_loaded = true; - let init_function = session.load_function(&module.self_id(), init_func_name, &[]); + let init_function = + session.load_function(&DummyCodeStorage, &module.self_id(), init_func_name, &[]); // it is ok to not have init_module function // init_module function should be (1) private and (2) has no return value // Note that for historic reasons, verification here is treated @@ -1384,6 +1410,7 @@ impl AptosVM { args, gas_meter, traversal_context, + &DummyCodeStorage, )?; } else { return Err(PartialVMError::new(StatusCode::CONSTRAINT_NOT_SATISFIED) @@ -1495,6 +1522,7 @@ impl AptosVM { .collect::>(); session.check_dependencies_and_charge_gas( + &DummyCodeStorage, gas_meter, traversal_context, modules @@ -1543,6 +1571,7 @@ impl AptosVM { bundle.into_inner(), destination, gas_meter, + &DummyCodeStorage, Compatibility::new( true, !self @@ -2137,6 +2166,7 @@ impl AptosVM { args, &mut gas_meter, &mut TraversalContext::new(&storage), + &DummyCodeStorage, ) .map(|_return_vals| ()) .or_else(|e| { @@ -2216,6 +2246,7 @@ impl AptosVM { serialize_values(&args), &mut gas_meter, &mut TraversalContext::new(&storage), + &DummyCodeStorage, ) .map(|_return_vals| ()) .or_else(|e| { @@ -2306,7 +2337,7 @@ impl AptosVM { arguments: Vec>, gas_meter: &mut impl AptosGasMeter, ) -> anyhow::Result>> { - let func = session.load_function(&module_id, &func_name, &type_args)?; + let func = session.load_function(&DummyCodeStorage, &module_id, &func_name, &type_args)?; let metadata = vm.extract_module_metadata(&module_id); let arguments = verifier::view_function::validate_view_function( session, @@ -2327,6 +2358,7 @@ impl AptosVM { arguments, gas_meter, &mut TraversalContext::new(&storage), + &DummyCodeStorage, ) .map_err(|err| anyhow!("Failed to execute function: {:?}", err))? .return_values @@ -2771,6 +2803,7 @@ fn create_account_if_does_not_exist( serialize_values(&vec![MoveValue::Address(account)]), gas_meter, traversal_context, + &DummyCodeStorage, ) .map(|_return_vals| ()) } diff --git a/aptos-move/aptos-vm/src/move_vm_ext/session/mod.rs b/aptos-move/aptos-vm/src/move_vm_ext/session/mod.rs index 2d39de276c7a8..296ae6fa8e987 100644 --- a/aptos-move/aptos-vm/src/move_vm_ext/session/mod.rs +++ b/aptos-move/aptos-vm/src/move_vm_ext/session/mod.rs @@ -288,6 +288,8 @@ impl<'r, 'l> SessionExt<'r, 'l> { let (modules, resources) = account_changeset.into_inner(); for (struct_tag, blob_op) in resources { + // TODO(George): Use ModuleStorage to resolve module metadata directly. + #[allow(deprecated)] let resource_group_tag = runtime .with_module_metadata(&struct_tag.module_id(), |md| { get_resource_group_member_from_metadata(&struct_tag, md) diff --git a/aptos-move/aptos-vm/src/move_vm_ext/warm_vm_cache.rs b/aptos-move/aptos-vm/src/move_vm_ext/warm_vm_cache.rs index 5f7b47ca41edc..271845f5f40cf 100644 --- a/aptos-move/aptos-vm/src/move_vm_ext/warm_vm_cache.rs +++ b/aptos-move/aptos-vm/src/move_vm_ext/warm_vm_cache.rs @@ -109,6 +109,7 @@ impl WarmVmCache { // // Loading up `0x1::account` should be sufficient as this is the most common module // used for prologue, epilogue and transfer functionality. + #[allow(deprecated)] let _ = vm.load_module( &ModuleId::new(CORE_CODE_ADDRESS, ident_str!("account").to_owned()), resolver, diff --git a/aptos-move/aptos-vm/src/transaction_validation.rs b/aptos-move/aptos-vm/src/transaction_validation.rs index 229833b71f01b..590aa9637eecc 100644 --- a/aptos-move/aptos-vm/src/transaction_validation.rs +++ b/aptos-move/aptos-vm/src/transaction_validation.rs @@ -27,7 +27,9 @@ use move_core_types::{ value::{serialize_values, MoveValue}, vm_status::{AbortLocation, StatusCode, VMStatus}, }; -use move_vm_runtime::{logging::expect_no_verification_errors, module_traversal::TraversalContext}; +use move_vm_runtime::{ + logging::expect_no_verification_errors, module_traversal::TraversalContext, DummyCodeStorage, +}; use move_vm_types::gas::UnmeteredGasMeter; use once_cell::sync::Lazy; @@ -216,6 +218,7 @@ pub(crate) fn run_script_prologue( serialize_values(&args), &mut gas_meter, traversal_context, + &DummyCodeStorage, ) .map(|_return_vals| ()) .map_err(expect_no_verification_errors) @@ -259,6 +262,7 @@ pub(crate) fn run_multisig_prologue( ]), &mut UnmeteredGasMeter, traversal_context, + &DummyCodeStorage, ) .map(|_return_vals| ()) .map_err(expect_no_verification_errors) @@ -317,6 +321,7 @@ fn run_epilogue( serialize_values(&args), &mut UnmeteredGasMeter, traversal_context, + &DummyCodeStorage, ) } else { // Regular tx, run the normal epilogue @@ -352,6 +357,7 @@ fn run_epilogue( serialize_values(&args), &mut UnmeteredGasMeter, traversal_context, + &DummyCodeStorage, ) } .map(|_return_vals| ()) @@ -380,6 +386,7 @@ fn emit_fee_statement( vec![bcs::to_bytes(&fee_statement).expect("Failed to serialize fee statement")], &mut UnmeteredGasMeter, traversal_context, + &DummyCodeStorage, ) .map(|_return_vals| ()) } diff --git a/aptos-move/aptos-vm/src/validator_txns/dkg.rs b/aptos-move/aptos-vm/src/validator_txns/dkg.rs index a0a57cdb04bcc..9d2e159c5af23 100644 --- a/aptos-move/aptos-vm/src/validator_txns/dkg.rs +++ b/aptos-move/aptos-vm/src/validator_txns/dkg.rs @@ -25,7 +25,10 @@ use move_core_types::{ value::{serialize_values, MoveValue}, vm_status::{AbortLocation, StatusCode, VMStatus}, }; -use move_vm_runtime::module_traversal::{TraversalContext, TraversalStorage}; +use move_vm_runtime::{ + module_traversal::{TraversalContext, TraversalStorage}, + DummyCodeStorage, +}; use move_vm_types::gas::UnmeteredGasMeter; #[derive(Debug)] @@ -114,6 +117,7 @@ impl AptosVM { serialize_values(&args), &mut gas_meter, &mut TraversalContext::new(&module_storage), + &DummyCodeStorage, ) .map_err(|e| { expect_only_successful_execution(e, FINISH_WITH_DKG_RESULT.as_str(), log_context) diff --git a/aptos-move/aptos-vm/src/validator_txns/jwk.rs b/aptos-move/aptos-vm/src/validator_txns/jwk.rs index 1d6e80f1e5cb6..17b4b2c18e6a8 100644 --- a/aptos-move/aptos-vm/src/validator_txns/jwk.rs +++ b/aptos-move/aptos-vm/src/validator_txns/jwk.rs @@ -31,7 +31,10 @@ use move_core_types::{ value::{serialize_values, MoveValue}, vm_status::{AbortLocation, StatusCode, VMStatus}, }; -use move_vm_runtime::module_traversal::{TraversalContext, TraversalStorage}; +use move_vm_runtime::{ + module_traversal::{TraversalContext, TraversalStorage}, + DummyCodeStorage, +}; use move_vm_types::gas::UnmeteredGasMeter; use std::collections::HashMap; @@ -144,6 +147,7 @@ impl AptosVM { serialize_values(&args), &mut gas_meter, &mut TraversalContext::new(&module_storage), + &DummyCodeStorage, ) .map_err(|e| { expect_only_successful_execution(e, UPSERT_INTO_OBSERVED_JWKS.as_str(), log_context) diff --git a/aptos-move/aptos-vm/src/verifier/event_validation.rs b/aptos-move/aptos-vm/src/verifier/event_validation.rs index 761a94bb2652d..e680ae357f1e1 100644 --- a/aptos-move/aptos-vm/src/verifier/event_validation.rs +++ b/aptos-move/aptos-vm/src/verifier/event_validation.rs @@ -120,13 +120,16 @@ pub(crate) fn extract_event_metadata_from_module( session: &mut SessionExt, module_id: &ModuleId, ) -> VMResult> { - let metadata = session.load_module(module_id).map(|module| { - CompiledModule::deserialize_with_config( - &module, - &session.get_vm_config().deserializer_config, - ) - .map(|module| aptos_framework::get_metadata_from_compiled_module(&module)) - }); + #[allow(deprecated)] + let metadata = session + .fetch_module_from_data_store(module_id) + .map(|module| { + CompiledModule::deserialize_with_config( + &module, + &session.get_vm_config().deserializer_config, + ) + .map(|module| aptos_framework::get_metadata_from_compiled_module(&module)) + }); if let Ok(Ok(Some(metadata))) = metadata { extract_event_metadata(&metadata) diff --git a/aptos-move/aptos-vm/src/verifier/randomness.rs b/aptos-move/aptos-vm/src/verifier/randomness.rs index e2360cd38b29e..65910c300d1e1 100644 --- a/aptos-move/aptos-vm/src/verifier/randomness.rs +++ b/aptos-move/aptos-vm/src/verifier/randomness.rs @@ -11,6 +11,7 @@ pub(crate) fn get_randomness_annotation( session: &mut SessionExt, entry_fn: &EntryFunction, ) -> VMResult> { + #[allow(deprecated)] let module = session .get_move_vm() .load_module(entry_fn.module(), resolver)?; diff --git a/aptos-move/aptos-vm/src/verifier/resource_groups.rs b/aptos-move/aptos-vm/src/verifier/resource_groups.rs index a7c7e8d193682..6cd2d2ce18491 100644 --- a/aptos-move/aptos-vm/src/verifier/resource_groups.rs +++ b/aptos-move/aptos-vm/src/verifier/resource_groups.rs @@ -150,12 +150,15 @@ pub(crate) fn extract_resource_group_metadata_from_module( BTreeMap, BTreeSet, )> { - let module = session.load_module(module_id).map(|module| { - CompiledModule::deserialize_with_config( - &module, - &session.get_vm_config().deserializer_config, - ) - }); + #[allow(deprecated)] + let module = session + .fetch_module_from_data_store(module_id) + .map(|module| { + CompiledModule::deserialize_with_config( + &module, + &session.get_vm_config().deserializer_config, + ) + }); let (metadata, module) = if let Ok(Ok(module)) = module { ( aptos_framework::get_metadata_from_compiled_module(&module), diff --git a/aptos-move/aptos-vm/src/verifier/transaction_arg_validation.rs b/aptos-move/aptos-vm/src/verifier/transaction_arg_validation.rs index 2ee495c5b10a6..abab0655bcfcf 100644 --- a/aptos-move/aptos-vm/src/verifier/transaction_arg_validation.rs +++ b/aptos-move/aptos-vm/src/verifier/transaction_arg_validation.rs @@ -22,7 +22,7 @@ use move_core_types::{ }; use move_vm_runtime::{ module_traversal::{TraversalContext, TraversalStorage}, - LoadedFunction, + DummyCodeStorage, LoadedFunction, }; use move_vm_types::{ gas::{GasMeter, UnmeteredGasMeter}, @@ -199,12 +199,12 @@ pub(crate) fn is_valid_txn_arg( match ty { Bool | U8 | U16 | U32 | U64 | U128 | U256 | Address => true, Vector(inner) => is_valid_txn_arg(session, inner, allowed_structs), - Struct { idx, .. } | StructInstantiation { idx, .. } => { - session.get_struct_type(*idx).is_some_and(|st| { + Struct { idx, .. } | StructInstantiation { idx, .. } => session + .fetch_struct_ty_by_idx(*idx, &DummyCodeStorage) + .is_some_and(|st| { let full_name = format!("{}::{}", st.module.short_str_lossless(), st.name); allowed_structs.contains_key(&full_name) - }) - }, + }), Signer | Reference(_) | MutableReference(_) | TyParam(_) => false, } } @@ -326,7 +326,7 @@ pub(crate) fn recursively_construct_arg( }, Struct { idx, .. } | StructInstantiation { idx, .. } => { let st = session - .get_struct_type(*idx) + .fetch_struct_ty_by_idx(*idx, &DummyCodeStorage) .ok_or_else(invalid_signature)?; let full_name = format!("{}::{}", st.module.short_str_lossless(), st.name); @@ -422,6 +422,7 @@ fn validate_and_construct( } let function = session.load_function_with_type_arg_inference( + &DummyCodeStorage, &constructor.module_id, constructor.func_name, expected_type, @@ -452,6 +453,7 @@ fn validate_and_construct( args, gas_meter, &mut TraversalContext::new(&storage), + &DummyCodeStorage, )?; let mut ret_vals = serialized_result.return_values; // We know ret_vals.len() == 1 diff --git a/aptos-move/e2e-tests/src/executor.rs b/aptos-move/e2e-tests/src/executor.rs index ea473cc7549d5..fb325138420ac 100644 --- a/aptos-move/e2e-tests/src/executor.rs +++ b/aptos-move/e2e-tests/src/executor.rs @@ -71,7 +71,10 @@ use move_core_types::{ language_storage::{ModuleId, StructTag, TypeTag}, move_resource::{MoveResource, MoveStructType}, }; -use move_vm_runtime::module_traversal::{TraversalContext, TraversalStorage}; +use move_vm_runtime::{ + module_traversal::{TraversalContext, TraversalStorage}, + DummyCodeStorage, +}; use move_vm_types::gas::UnmeteredGasMeter; use serde::Serialize; use std::{ @@ -985,7 +988,12 @@ impl FakeExecutor { let mut session = vm.new_session(&resolver, SessionId::void(), None); // load function name into cache to ensure cache is hot - let _ = session.load_function(module, &Self::name(function_name), &type_params.clone()); + let _ = session.load_function( + &DummyCodeStorage, + module, + &Self::name(function_name), + &type_params.clone(), + ); let fun_name = Self::name(function_name); let should_error = fun_name.clone().into_string().ends_with(POSTFIX); @@ -1029,6 +1037,7 @@ impl FakeExecutor { arg, regular.as_mut().unwrap(), &mut TraversalContext::new(&storage), + &DummyCodeStorage, ), GasMeterType::UnmeteredGasMeter => session.execute_function_bypass_visibility( module, @@ -1037,6 +1046,7 @@ impl FakeExecutor { arg, unmetered.as_mut().unwrap(), &mut TraversalContext::new(&storage), + &DummyCodeStorage, ), }; let elapsed = start.elapsed(); @@ -1111,6 +1121,7 @@ impl FakeExecutor { shared_buffer: Arc::clone(&a1), }), &mut TraversalContext::new(&storage), + &DummyCodeStorage, ); if let Err(err) = result { if !should_error { @@ -1164,6 +1175,7 @@ impl FakeExecutor { // TODO(Gas): we probably want to switch to metered execution in the future &mut UnmeteredGasMeter, &mut TraversalContext::new(&storage), + &DummyCodeStorage, ) .unwrap_or_else(|e| { panic!( @@ -1222,6 +1234,7 @@ impl FakeExecutor { // TODO(Gas): we probably want to switch to metered execution in the future &mut UnmeteredGasMeter, &mut TraversalContext::new(&storage), + &DummyCodeStorage, ) .map_err(|e| e.into_vm_status())?; diff --git a/aptos-move/framework/src/module_metadata.rs b/aptos-move/framework/src/module_metadata.rs index 18a6178e23bc6..1fdf891034139 100644 --- a/aptos-move/framework/src/module_metadata.rs +++ b/aptos-move/framework/src/module_metadata.rs @@ -228,6 +228,8 @@ pub fn get_metadata_v0(md: &[Metadata]) -> Option> /// Extract metadata from the VM, upgrading V0 to V1 representation as needed pub fn get_vm_metadata(vm: &MoveVM, module_id: &ModuleId) -> Option> { + // TODO(George): Use ModuleStorage to resolve module metadata directly. + #[allow(deprecated)] vm.with_module_metadata(module_id, get_metadata) } @@ -236,6 +238,8 @@ pub fn get_vm_metadata_v0( vm: &MoveVM, module_id: &ModuleId, ) -> Option> { + // TODO(George): Use ModuleStorage to resolve module metadata directly. + #[allow(deprecated)] vm.with_module_metadata(module_id, get_metadata_v0) } diff --git a/aptos-move/vm-genesis/src/lib.rs b/aptos-move/vm-genesis/src/lib.rs index aa36b820ada63..f0694eee9d060 100644 --- a/aptos-move/vm-genesis/src/lib.rs +++ b/aptos-move/vm-genesis/src/lib.rs @@ -49,7 +49,10 @@ use move_core_types::{ language_storage::{ModuleId, TypeTag}, value::{serialize_values, MoveTypeLayout, MoveValue}, }; -use move_vm_runtime::module_traversal::{TraversalContext, TraversalStorage}; +use move_vm_runtime::{ + module_traversal::{TraversalContext, TraversalStorage}, + DummyCodeStorage, +}; use move_vm_types::gas::UnmeteredGasMeter; use once_cell::sync::Lazy; use rand::prelude::*; @@ -382,6 +385,7 @@ fn exec_function( args, &mut UnmeteredGasMeter, &mut TraversalContext::new(&storage), + &DummyCodeStorage, ) .unwrap_or_else(|e| { panic!( @@ -763,7 +767,7 @@ fn publish_package(session: &mut SessionExt, pack: &ReleasePackage) { .map(|(c, _)| c.to_vec()) .collect::>(); session - .publish_module_bundle(code, addr, &mut UnmeteredGasMeter) + .publish_module_bundle(code, addr, &mut UnmeteredGasMeter, &DummyCodeStorage) .unwrap_or_else(|e| { panic!( "Failure publishing package `{}`: {:?}", diff --git a/third_party/move/extensions/async/move-async-vm/src/async_vm.rs b/third_party/move/extensions/async/move-async-vm/src/async_vm.rs new file mode 100644 index 0000000000000..0c5c350686033 --- /dev/null +++ b/third_party/move/extensions/async/move-async-vm/src/async_vm.rs @@ -0,0 +1,455 @@ +// Copyright (c) The Diem Core Contributors +// Copyright (c) The Move Contributors +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + actor_metadata, + actor_metadata::ActorMetadata, + natives, + natives::{AsyncExtension, GasParameters as ActorGasParameters}, +}; +use move_binary_format::errors::{Location, PartialVMError, PartialVMResult, VMError, VMResult}; +use move_core_types::{ + account_address::AccountAddress, + effects::{ChangeSet, Op}, + identifier::Identifier, + language_storage::{ModuleId, StructTag, TypeTag}, + vm_status::StatusCode, +}; +use move_vm_runtime::{ + module_traversal::*, + move_vm::MoveVM, + native_extensions::NativeContextExtensions, + native_functions::NativeFunction, + session::{SerializedReturnValues, Session}, + DummyCodeStorage, +}; +use move_vm_test_utils::gas_schedule::{Gas, GasStatus}; +use move_vm_types::{ + resolver::MoveResolver, + values::{Reference, Value}, +}; +use std::{ + collections::HashMap, + error::Error, + fmt::{Debug, Display, Formatter}, +}; + +/// Represents an instance of an async VM. +pub struct AsyncVM { + move_vm: MoveVM, + actor_metadata: HashMap, + message_table: HashMap, +} + +impl AsyncVM { + /// Creates a new VM, registering the given natives and actors. + pub fn new( + async_lib_addr: AccountAddress, + natives: I, + actors: A, + actor_gas_parameters: ActorGasParameters, + ) -> VMResult + where + I: IntoIterator, + A: IntoIterator, + { + let actor_metadata: HashMap = actors + .into_iter() + .map(|a| (a.module_id.clone(), a)) + .collect(); + let message_table: HashMap = actor_metadata + .values() + .flat_map(|a| { + a.messages.iter().map(move |m| { + ( + actor_metadata::message_hash(&a.module_id, m.as_ident_str()), + (a.module_id.clone(), m.clone()), + ) + }) + }) + .collect(); + Ok(AsyncVM { + move_vm: MoveVM::new( + natives + .into_iter() + .chain(natives::actor_natives(async_lib_addr, actor_gas_parameters)), + ), + actor_metadata, + message_table, + }) + } + + /// Creates a new session. + pub fn new_session<'r, 'l>( + &'l self, + for_actor: AccountAddress, + virtual_time: u128, + move_resolver: &'r impl MoveResolver, + ) -> AsyncSession<'r, 'l> { + self.new_session_with_extensions( + for_actor, + virtual_time, + move_resolver, + NativeContextExtensions::default(), + ) + } + + /// Creates a new session. + pub fn new_session_with_extensions<'r, 'l>( + &'l self, + for_actor: AccountAddress, + virtual_time: u128, + move_resolver: &'r impl MoveResolver, + ext: NativeContextExtensions<'r>, + ) -> AsyncSession<'r, 'l> { + let extensions = make_extensions(ext, for_actor, virtual_time, true); + AsyncSession { + vm: self, + vm_session: self + .move_vm + .new_session_with_extensions(move_resolver, extensions), + } + } + + /// Get the underlying Move VM. + pub fn get_move_vm(&mut self) -> &mut MoveVM { + &mut self.move_vm + } + + /// Resolve the message hash into module and handler function. + pub fn resolve_message_hash(&self, message_hash: u64) -> Option<&(ModuleId, Identifier)> { + self.message_table.get(&message_hash) + } + + /// Get the actor metadata. + pub fn actor_metadata(&self, module_id: &ModuleId) -> Option<&ActorMetadata> { + self.actor_metadata.get(module_id) + } + + /// Get all know actors. + pub fn actors(&self) -> Vec { + self.actor_metadata.keys().cloned().collect() + } +} + +/// Represents an Async Move execution session. +pub struct AsyncSession<'r, 'l> { + vm: &'l AsyncVM, + vm_session: Session<'r, 'l>, +} + +/// Represents a message being sent, consisting of target address, message hash, and arguments. +pub type Message = (AccountAddress, u64, Vec>); + +/// A structure to represent success for the execution of an async session operation. +pub struct AsyncSuccess<'r> { + pub change_set: ChangeSet, + pub messages: Vec, + pub gas_used: Gas, + pub ext: NativeContextExtensions<'r>, +} + +/// A structure to represent failure for the execution of an async session operation. +#[derive(Debug, Clone)] +pub struct AsyncError { + pub error: VMError, + pub gas_used: Gas, +} + +/// Result type for operations of an AsyncSession. +pub type AsyncResult<'r> = Result, AsyncError>; + +impl<'r, 'l> AsyncSession<'r, 'l> { + /// Get the underlying Move VM session. + pub fn get_move_session(&mut self) -> &mut Session<'r, 'l> { + &mut self.vm_session + } + + /// Creates a new actor, identified by the module_id, at the given account address. + /// This calls the initializer function of the actor, and returns on success + /// a changeset which needs to be committed to persist the new actors state. + pub fn new_actor( + mut self, + module_id: &ModuleId, + actor_addr: AccountAddress, + gas_status: &mut GasStatus, + ) -> AsyncResult<'r> { + let actor = self + .vm + .actor_metadata + .get(module_id) + .ok_or_else(|| async_extension_error(format!("actor `{}` unknown", module_id)))?; + let state_type_tag = TypeTag::Struct(Box::new(actor.state_tag.clone())); + let state_type = self + .vm_session + .load_type(&state_type_tag, &DummyCodeStorage) + .map_err(vm_error_to_async)?; + + // Check whether the actor state already exists. + let state = self + .vm_session + .load_resource(&DummyCodeStorage, actor_addr, &state_type) + .map(|(gv, _)| gv) + .map_err(partial_vm_error_to_async)?; + if state.exists().map_err(partial_vm_error_to_async)? { + return Err(async_extension_error(format!( + "actor `{}` already exists at `{}`", + module_id.short_str_lossless(), + actor_addr.short_str_lossless() + ))); + } + + // Execute the initializer. + let gas_before = gas_status.remaining_gas(); + let traversal_storage = TraversalStorage::new(); + let result = self + .vm_session + .execute_function_bypass_visibility( + &actor.module_id, + &actor.initializer, + vec![], + Vec::>::new(), + gas_status, + &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + ) + .and_then(|ret| Ok((ret, self.vm_session.finish_with_extensions()?))); + let gas_used = gas_before.checked_sub(gas_status.remaining_gas()).unwrap(); + + // Process the result, moving the return value of the initializer function into the + // changeset. + match result { + Ok(( + SerializedReturnValues { + mutable_reference_outputs: _, + mut return_values, + }, + (mut change_set, mut native_extensions), + )) => { + if return_values.len() != 1 { + Err(async_extension_error(format!( + "inconsistent initializer `{}`", + actor.initializer + ))) + } else { + publish_actor_state( + &mut change_set, + actor_addr, + actor.state_tag.clone(), + return_values.remove(0).0, + false, + ) + .map_err(partial_vm_error_to_async)?; + let async_ext = native_extensions.remove::(); + Ok(AsyncSuccess { + change_set, + messages: async_ext.sent, + gas_used, + ext: native_extensions, + }) + } + }, + Err(error) => Err(AsyncError { error, gas_used }), + } + } + + /// Handles a message at `actor` with the given `message_hash`. This will call the + /// according function as determined by the AsyncResolver, passing a reference to + /// the actors state. + pub fn handle_message( + mut self, + actor_addr: AccountAddress, + message_hash: u64, + mut args: Vec>, + gas_status: &mut GasStatus, + ) -> AsyncResult<'r> { + // Resolve actor and function which handles the message. + let (module_id, handler_id) = + self.vm.message_table.get(&message_hash).ok_or_else(|| { + async_extension_error(format!("unknown message hash `{}`", message_hash)) + })?; + let actor = self.vm.actor_metadata.get(module_id).ok_or_else(|| { + async_extension_error(format!( + "actor `{}` unknown", + module_id.short_str_lossless() + )) + })?; + + // Load the resource representing the actor state and add to arguments. + let state_type_tag = TypeTag::Struct(Box::new(actor.state_tag.clone())); + let state_type = self + .vm_session + .load_type(&state_type_tag, &DummyCodeStorage) + .map_err(vm_error_to_async)?; + + let actor_state_global = self + .vm_session + .load_resource(&DummyCodeStorage, actor_addr, &state_type) + .map(|(gv, _)| gv) + .map_err(partial_vm_error_to_async)?; + let actor_state = actor_state_global + .borrow_global() + .and_then(|v| v.value_as::()) + .and_then(|r| r.read_ref()) + .map_err(partial_vm_error_to_async)?; + args.insert( + 0, + self.to_bcs(actor_state, &state_type_tag) + .map_err(partial_vm_error_to_async)?, + ); + + // Execute the handler. + let gas_before = gas_status.remaining_gas(); + let traversal_storage = TraversalStorage::new(); + let result = self + .vm_session + .execute_function_bypass_visibility( + module_id, + handler_id, + vec![], + args, + gas_status, + &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + ) + .and_then(|ret| Ok((ret, self.vm_session.finish_with_extensions()?))); + + let gas_used = gas_before.checked_sub(gas_status.remaining_gas()).unwrap(); + + // Process the result, moving the mutated value of the handlers first parameter + // into the changeset. + match result { + Ok(( + SerializedReturnValues { + mut mutable_reference_outputs, + return_values: _, + }, + (mut change_set, mut native_extensions), + )) => { + if mutable_reference_outputs.len() > 1 { + Err(async_extension_error(format!( + "inconsistent handler `{}`", + handler_id + ))) + } else { + if !mutable_reference_outputs.is_empty() { + publish_actor_state( + &mut change_set, + actor_addr, + actor.state_tag.clone(), + mutable_reference_outputs.remove(0).1, + true, + ) + .map_err(partial_vm_error_to_async)?; + } + let async_ext = native_extensions.remove::(); + Ok(AsyncSuccess { + change_set, + messages: async_ext.sent, + gas_used, + ext: native_extensions, + }) + } + }, + Err(error) => Err(AsyncError { error, gas_used }), + } + } + + #[allow(clippy::wrong_self_convention)] + fn to_bcs(&mut self, value: Value, tag: &TypeTag) -> PartialVMResult> { + let type_layout = self + .vm_session + .get_type_layout(tag, &DummyCodeStorage) + .map_err(|e| e.to_partial())?; + value + .simple_serialize(&type_layout) + .ok_or_else(|| partial_extension_error("serialization failed")) + } +} + +fn make_extensions( + mut exts: NativeContextExtensions, + actor_addr: AccountAddress, + virtual_time: u128, + in_initializer: bool, +) -> NativeContextExtensions { + exts.add(AsyncExtension { + current_actor: actor_addr, + sent: vec![], + in_initializer, + virtual_time, + }); + exts +} + +fn publish_actor_state( + change_set: &mut ChangeSet, + actor_addr: AccountAddress, + state_tag: StructTag, + state: Vec, + is_modify: bool, +) -> PartialVMResult<()> { + change_set + .add_resource_op( + actor_addr, + state_tag, + if is_modify { + Op::Modify(state.into()) + } else { + Op::New(state.into()) + }, + ) + .map_err(|err| partial_extension_error(format!("cannot publish actor state: {}", err))) +} + +pub(crate) fn partial_extension_error(msg: impl ToString) -> PartialVMError { + PartialVMError::new(StatusCode::VM_EXTENSION_ERROR).with_message(msg.to_string()) +} + +pub(crate) fn extension_error(msg: impl ToString) -> VMError { + partial_extension_error(msg).finish(Location::Undefined) +} + +fn async_extension_error(msg: impl ToString) -> AsyncError { + AsyncError { + error: extension_error(msg), + gas_used: 0.into(), + } +} + +fn vm_error_to_async(error: VMError) -> AsyncError { + AsyncError { + error, + gas_used: 0.into(), + } +} + +fn partial_vm_error_to_async(error: PartialVMError) -> AsyncError { + vm_error_to_async(error.finish(Location::Undefined)) +} + +// ------------------------------------------------------------------------------------------ +// Displaying + +impl Display for AsyncError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.error) + } +} + +impl Error for AsyncError {} + +impl<'r> Display for AsyncSuccess<'r> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let AsyncSuccess { + change_set, + messages, + gas_used, + ext: _, + } = self; + write!(f, "change_set: {:?}", change_set)?; + write!(f, ", messages: {:?}", messages)?; + write!(f, ", gas: {}", gas_used) + } +} diff --git a/third_party/move/extensions/async/move-async-vm/tests/testsuite.rs b/third_party/move/extensions/async/move-async-vm/tests/testsuite.rs new file mode 100644 index 0000000000000..e7b2cf68a75d3 --- /dev/null +++ b/third_party/move/extensions/async/move-async-vm/tests/testsuite.rs @@ -0,0 +1,429 @@ +// Copyright (c) The Diem Core Contributors +// Copyright (c) The Move Contributors +// SPDX-License-Identifier: Apache-2.0 + +use bytes::Bytes; +use itertools::Itertools; +use move_async_vm::{ + actor_metadata, + actor_metadata::ActorMetadata, + async_vm::{AsyncResult, AsyncSession, AsyncVM, Message}, + natives::GasParameters as ActorGasParameters, +}; +use move_binary_format::{access::ModuleAccess, errors::PartialVMResult}; +use move_command_line_common::testing::get_compiler_exp_extension; +use move_compiler::{ + attr_derivation, compiled_unit::CompiledUnit, diagnostics::report_diagnostics_to_buffer, + shared::NumericalAddress, Compiler, Flags, +}; +use move_core_types::{ + account_address::AccountAddress, + effects::{ChangeSet, Op}, + ident_str, + identifier::{IdentStr, Identifier}, + language_storage::{ModuleId, StructTag}, + metadata::Metadata, + value::MoveTypeLayout, +}; +use move_prover_test_utils::{baseline_test::verify_or_update_baseline, extract_test_directives}; +use move_vm_runtime::DummyCodeStorage; +use move_vm_test_utils::gas_schedule::GasStatus; +use move_vm_types::resolver::{resource_size, ModuleResolver, ResourceResolver}; +use std::{ + cell::RefCell, + collections::{BTreeMap, BTreeSet, VecDeque}, + path::{Path, PathBuf}, + str::FromStr, +}; + +const TEST_ADDR: &str = "0x3"; +const SOURCE_DIRS: &[&str] = &[ + "./tests/sources", + "../move-async-lib/sources", + "../../../move-stdlib/sources", +]; + +struct Harness { + module_cache: BTreeMap, + vm: AsyncVM, + actor_instances: Vec<(ModuleId, AccountAddress)>, + baseline: RefCell, + resource_store: RefCell>, +} + +fn test_account() -> AccountAddress { + AccountAddress::from_hex_literal(TEST_ADDR).expect("valid test address") +} + +fn test_runner(path: &Path) -> datatest_stable::Result<()> { + let target_module = path + .with_extension("") + .file_name() + .unwrap() + .to_string_lossy() + .to_string(); + let deps = extract_test_directives(path, "// dep:")?; + let actors = extract_test_directives(path, "// actor: ")?; + let instances = extract_test_directives(path, "// instance: ")?; + let harness = Harness::new( + std::iter::once(target_module.clone()) + .chain(deps.into_iter()) + .collect(), + actors, + instances, + )?; + harness.run(&target_module)?; + let baseline_path = path.with_extension(get_compiler_exp_extension()); + verify_or_update_baseline(baseline_path.as_path(), harness.baseline.borrow().as_str())?; + Ok(()) +} + +datatest_stable::harness!(test_runner, "tests/sources", r".*\.move$"); + +// ======================================================================================== +// Test execution + +impl Harness { + fn run(&self, _module: &str) -> anyhow::Result<()> { + let mut gas = GasStatus::new_unmetered(); + let mut tick = 0; + // Publish modules. + let proxy = HarnessProxy { harness: self }; + let mut session = self.vm.new_session(test_account(), 0, &proxy); + let mut done = BTreeSet::new(); + for id in self.module_cache.keys() { + self.publish_module(&mut session, id, &mut gas, &mut done)?; + } + // Initialize actors + let mut mailbox: VecDeque = Default::default(); + for (actor, addr) in self.actor_instances.clone() { + self.log(format!( + "actor 0x{} created from {}", + addr.short_str_lossless(), + actor.short_str_lossless() + )); + { + let proxy = HarnessProxy { harness: self }; + let session = self.vm.new_session(addr, 0, &proxy); + let result = session.new_actor(&actor, addr, &mut gas); + self.handle_result(&mut mailbox, result); + }; + + // Put a start message for this actor into the mailbox. + let entry_point_id = ident_str!("start"); + let hash = actor_metadata::message_hash(&actor, entry_point_id); + mailbox.push_back((addr, hash, vec![])); + } + + // Handle messages until the mailbox is empty. + while let Some((actor, message_hash, args)) = mailbox.pop_front() { + // Baseline logging + if let Some((module_id, fun_id)) = self.vm.resolve_message_hash(message_hash).cloned() { + self.log(format!( + "actor 0x{} handling {}::{} (hash=0x{:X})", + actor.short_str_lossless(), + module_id.short_str_lossless(), + fun_id, + message_hash + )); + } else { + self.log(format!( + "actor 0x{} handling ???? (hash={})", + actor.short_str_lossless(), + message_hash + )) + } + // Handling + let proxy = HarnessProxy { harness: self }; + let session = self.vm.new_session(actor, tick, &proxy); + tick += 1000_1000; // micros + let result = session.handle_message(actor, message_hash, args, &mut gas); + self.handle_result(&mut mailbox, result); + } + Ok(()) + } + + fn publish_module( + &self, + session: &mut AsyncSession, + id: &IdentStr, + gas: &mut GasStatus, + done: &mut BTreeSet, + ) -> anyhow::Result<()> { + if done.insert(id.to_owned()) { + let cu = self.module_cache.get(id).unwrap(); + if let CompiledUnit::Module(m) = cu { + for dep in &m.module.module_handles { + let dep_id = m.module.identifier_at(dep.name); + self.publish_module(session, dep_id, gas, done)? + } + } + self.log(format!("publishing {}", id)); + session.get_move_session().publish_module( + cu.serialize(None), + test_account(), + gas, + &DummyCodeStorage, + )? + } + Ok(()) + } + + fn handle_result(&self, mailbox: &mut VecDeque, result: AsyncResult) { + match result { + Ok(success) => { + self.log(" SUCCESS"); + for m in &success.messages { + self.log(format!( + " sent 0x{} <- 0x{:X} argc={}", + m.0.short_str_lossless(), + m.1, + m.2.len() + )) + } + mailbox.extend(success.messages); + self.commit_changeset(success.change_set) + }, + Err(error) => self.log(format!(" FAIL {:}", error)), + } + } + + fn commit_changeset(&self, changeset: ChangeSet) { + for (addr, change) in changeset.into_inner() { + for (struct_tag, op) in change.into_inner().1 { + self.log(format!( + " commit 0x{}::{}::{}[0x{}] := {:?}", + struct_tag.address.short_str_lossless(), + struct_tag.module, + struct_tag.module, + addr.short_str_lossless(), + op.as_ref().map(|b| format!("{:02X?}", b)) + )); + match op { + Op::New(v) => { + assert!(self + .resource_store + .borrow_mut() + .insert((addr, struct_tag), v) + .is_none()); + }, + Op::Modify(v) => { + self.resource_store + .borrow_mut() + .insert((addr, struct_tag), v) + .unwrap(); + }, + Op::Delete => { + self.resource_store + .borrow_mut() + .remove(&(addr, struct_tag)) + .unwrap(); + }, + } + } + } + } +} + +// ======================================================================================== +// Harness creation + +impl Harness { + fn new( + modules: Vec, + actors: Vec, + instances: Vec, + ) -> anyhow::Result { + // Create address map. We are mapping all aliases to TEST_ADDR for simplicity. + let test_addr = NumericalAddress::parse_str(TEST_ADDR).unwrap(); + let address_map: BTreeMap = vec![ + ("std".to_string(), test_addr), + ("Async".to_string(), test_addr), + ("Test".to_string(), test_addr), + ] + .into_iter() + .collect(); + // Collect metadata and compile modules. + let actor_metadata = Self::collect_metadata(actors)?; + let actor_instances = Self::collect_instances(instances)?; + let module_files = Self::collect_modules(modules)?; + let module_cache = Self::compile(&address_map, &module_files)?; + let harness = Harness { + baseline: Default::default(), + module_cache, + resource_store: Default::default(), + vm: AsyncVM::new( + test_account(), + move_stdlib::natives::all_natives( + test_account(), + // We may want to switch to a different gas schedule in the future, but for now, + // the all-zero one should be enough. + move_stdlib::natives::GasParameters::zeros(), + ), + actor_metadata, + ActorGasParameters::zeros(), + )?, + actor_instances, + }; + Ok(harness) + } + + fn collect_metadata(actors: Vec) -> anyhow::Result> { + let mut actor_metadata = vec![]; + for actor in actors { + // format: 0x3 Module State init message.. + let parts = actor.split_ascii_whitespace().collect_vec(); + if parts.len() < 4 { + anyhow::bail!("malformed actor decl `{}`", actor) + } + let address = AccountAddress::from_hex_literal(parts[0])?; + let module = Identifier::from_str(parts[1])?; + let struct_ = Identifier::from_str(parts[2])?; + let initializer = Identifier::from_str(parts[3])?; + let state_tag = StructTag { + address, + module: module.clone(), + name: struct_, + type_args: vec![], + }; + let mut messages = vec![]; + for message in &parts[4..] { + messages.push(Identifier::from_str(message)?) + } + actor_metadata.push(ActorMetadata { + module_id: ModuleId::new(address, module), + state_tag, + initializer, + messages, + }) + } + Ok(actor_metadata) + } + + fn collect_instances( + instances: Vec, + ) -> anyhow::Result> { + let mut result = vec![]; + for inst in instances { + // format: 0x3 Module 0x23 + // where the last address is where the instance is to create + let parts = inst.split_ascii_whitespace().collect_vec(); + if parts.len() != 3 { + anyhow::bail!("malformed instance decl `{}`", inst) + } + let address = AccountAddress::from_hex_literal(parts[0])?; + let module = Identifier::from_str(parts[1])?; + let inst_address = AccountAddress::from_hex_literal(parts[2])?; + result.push((ModuleId::new(address, module), inst_address)) + } + Ok(result) + } + + fn collect_modules(deps: Vec) -> anyhow::Result> { + let mut module_files = BTreeMap::new(); + for dep in &deps { + let mut found = false; + for dir in SOURCE_DIRS { + let mut path = PathBuf::from(dir); + path.push(format!("{}.move", dep)); + if path.exists() { + module_files.insert( + Identifier::from_str(dep).expect("valid identifier"), + path.to_string_lossy().to_string(), + ); + found = true; + break; + } + } + if !found { + anyhow::bail!("dependency {} not found", dep) + } + } + Ok(module_files) + } + + fn compile( + address_map: &BTreeMap, + module_files: &BTreeMap, + ) -> anyhow::Result> { + let mut module_cache = BTreeMap::new(); + for (id, path) in module_files { + let targets = vec![path.to_owned()]; + let deps = module_files + .values() + .filter(|p| *p != path) + .cloned() + .collect(); + let flags = Flags::empty().set_flavor("async"); + let known_attributes = attr_derivation::get_known_attributes_for_flavor(&flags); + let compiler = + Compiler::from_files(targets, deps, address_map.clone(), flags, &known_attributes); + let (sources, inner) = compiler.build()?; + match inner { + Err(diags) => anyhow::bail!( + "Compilation failure {{\n{}\n}}", + String::from_utf8_lossy( + report_diagnostics_to_buffer(&sources, diags).as_slice() + ) + ), + Ok((mut units, _)) => { + module_cache.insert(id.to_owned(), units.remove(0).into_compiled_unit()); + }, + } + } + Ok(module_cache) + } +} + +// ======================================================================================== +// Move Resolver + +/// A facade for the harness which can appear as mutable, even though the harness +/// is not. Keeping the harness immutable and using RefCell for the few mutation points +/// simplifies things in this test. +struct HarnessProxy<'a> { + harness: &'a Harness, +} + +impl<'a> ModuleResolver for HarnessProxy<'a> { + fn get_module_metadata(&self, _module_id: &ModuleId) -> Vec { + vec![] + } + + fn get_module(&self, id: &ModuleId) -> PartialVMResult> { + Ok(self + .harness + .module_cache + .get(id.name()) + .map(|c| c.serialize(None).into())) + } +} + +impl<'a> ResourceResolver for HarnessProxy<'a> { + fn get_resource_bytes_with_metadata_and_layout( + &self, + address: &AccountAddress, + typ: &StructTag, + _metadata: &[Metadata], + _maybe_layout: Option<&MoveTypeLayout>, + ) -> PartialVMResult<(Option, usize)> { + let res = self + .harness + .resource_store + .borrow() + .get(&(*address, typ.clone())) + .cloned(); + let res_size = resource_size(&res); + Ok((res, res_size)) + } +} + +// ======================================================================================== +// Baseline writer + +impl Harness { + fn log(&self, s: impl ToString) { + let s = s.to_string(); + self.baseline.borrow_mut().push_str(&(s + "\n")) + } +} diff --git a/third_party/move/move-vm/integration-tests/src/tests/bad_entry_point_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/bad_entry_point_tests.rs index 40dff370a5012..dcac6fe6cd4d9 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/bad_entry_point_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/bad_entry_point_tests.rs @@ -10,7 +10,7 @@ use move_core_types::{ value::{serialize_values, MoveValue}, vm_status::StatusType, }; -use move_vm_runtime::{module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::{BlankStorage, InMemoryStorage}; use move_vm_types::gas::UnmeteredGasMeter; @@ -34,6 +34,7 @@ fn call_non_existent_module() { serialize_values(&vec![MoveValue::Signer(TEST_ADDR)]), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err(); @@ -71,6 +72,7 @@ fn call_non_existent_function() { serialize_values(&vec![MoveValue::Signer(TEST_ADDR)]), &mut UnmeteredGasMeter, &mut TraversalContext::new(&storage), + &DummyCodeStorage, ) .unwrap_err(); diff --git a/third_party/move/move-vm/integration-tests/src/tests/bad_storage_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/bad_storage_tests.rs index 64dc94029efd3..dad5d643cb60f 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/bad_storage_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/bad_storage_tests.rs @@ -13,7 +13,7 @@ use move_core_types::{ value::{serialize_values, MoveTypeLayout, MoveValue}, vm_status::{StatusCode, StatusType}, }; -use move_vm_runtime::{module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::{ gas::UnmeteredGasMeter, @@ -103,6 +103,8 @@ fn test_malformed_resource() { vec![MoveValue::Signer(TEST_ADDR).simple_serialize().unwrap()], &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .map(|_| ()) .unwrap(); @@ -122,6 +124,8 @@ fn test_malformed_resource() { vec![MoveValue::Signer(TEST_ADDR).simple_serialize().unwrap()], &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .map(|_| ()) .unwrap(); @@ -150,6 +154,8 @@ fn test_malformed_resource() { vec![MoveValue::Signer(TEST_ADDR).simple_serialize().unwrap()], &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .map(|_| ()) .unwrap_err(); @@ -191,6 +197,7 @@ fn test_malformed_module() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); } @@ -218,6 +225,7 @@ fn test_malformed_module() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err(); assert_eq!(err.status_type(), StatusType::InvariantViolation); @@ -259,6 +267,7 @@ fn test_unverifiable_module() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); } @@ -285,6 +294,7 @@ fn test_unverifiable_module() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err(); @@ -337,6 +347,7 @@ fn test_missing_module_dependency() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); } @@ -358,6 +369,7 @@ fn test_missing_module_dependency() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err(); @@ -410,6 +422,7 @@ fn test_malformed_module_dependency() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); } @@ -437,6 +450,7 @@ fn test_malformed_module_dependency() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err(); @@ -490,6 +504,7 @@ fn test_unverifiable_module_dependency() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); } @@ -517,6 +532,7 @@ fn test_unverifiable_module_dependency() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err(); @@ -609,6 +625,7 @@ fn test_storage_returns_bogus_error_when_loading_module() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err(); @@ -678,6 +695,7 @@ fn test_storage_returns_bogus_error_when_loading_resource() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); @@ -689,6 +707,7 @@ fn test_storage_returns_bogus_error_when_loading_resource() { serialize_values(&vec![MoveValue::Signer(TEST_ADDR)]), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err(); diff --git a/third_party/move/move-vm/integration-tests/src/tests/binary_format_version.rs b/third_party/move/move-vm/integration-tests/src/tests/binary_format_version.rs index ce7c616f03943..52ab2dc4717f8 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/binary_format_version.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/binary_format_version.rs @@ -7,7 +7,7 @@ use move_binary_format::{ file_format_common::{IDENTIFIER_SIZE_MAX, VERSION_MAX}, }; use move_core_types::{account_address::AccountAddress, vm_status::StatusCode}; -use move_vm_runtime::{config::VMConfig, module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{config::VMConfig, module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; @@ -34,6 +34,7 @@ fn test_publish_module_with_custom_max_binary_format_version() { b_new.clone(), *m.self_id().address(), &mut UnmeteredGasMeter, + &DummyCodeStorage, ) .unwrap(); @@ -41,6 +42,7 @@ fn test_publish_module_with_custom_max_binary_format_version() { b_old.clone(), *m.self_id().address(), &mut UnmeteredGasMeter, + &DummyCodeStorage, ) .unwrap(); } @@ -68,6 +70,7 @@ fn test_publish_module_with_custom_max_binary_format_version() { b_new.clone(), *m.self_id().address(), &mut UnmeteredGasMeter, + &DummyCodeStorage, ) .unwrap_err() .major_status(), @@ -78,6 +81,7 @@ fn test_publish_module_with_custom_max_binary_format_version() { b_old.clone(), *m.self_id().address(), &mut UnmeteredGasMeter, + &DummyCodeStorage, ) .unwrap(); } @@ -110,6 +114,8 @@ fn test_run_script_with_custom_max_binary_format_version() { args.clone(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .unwrap(); @@ -119,6 +125,8 @@ fn test_run_script_with_custom_max_binary_format_version() { args, &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .unwrap(); } @@ -148,7 +156,9 @@ fn test_run_script_with_custom_max_binary_format_version() { vec![], args.clone(), &mut UnmeteredGasMeter, - &mut TraversalContext::new(&traversal_storage) + &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .unwrap_err() .major_status(), @@ -161,6 +171,8 @@ fn test_run_script_with_custom_max_binary_format_version() { args, &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .unwrap(); } diff --git a/third_party/move/move-vm/integration-tests/src/tests/exec_func_effects_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/exec_func_effects_tests.rs index c9384cc307ba9..26fbd7aa39fa3 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/exec_func_effects_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/exec_func_effects_tests.rs @@ -13,7 +13,9 @@ use move_core_types::{ value::{serialize_values, MoveValue}, vm_status::StatusCode, }; -use move_vm_runtime::{module_traversal::*, move_vm::MoveVM, session::SerializedReturnValues}; +use move_vm_runtime::{ + module_traversal::*, move_vm::MoveVM, session::SerializedReturnValues, DummyCodeStorage, +}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; use std::convert::TryInto; @@ -106,6 +108,7 @@ fn run( serialize_values(&vec![arg_val0]), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .and_then(|ret_values| { let change_set = session.finish()?; diff --git a/third_party/move/move-vm/integration-tests/src/tests/function_arg_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/function_arg_tests.rs index b3576f6287944..6f0161e7aaa31 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/function_arg_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/function_arg_tests.rs @@ -12,7 +12,7 @@ use move_core_types::{ value::{MoveStruct, MoveValue}, vm_status::StatusCode, }; -use move_vm_runtime::{module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; @@ -77,6 +77,7 @@ fn run( args, &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, )?; Ok(()) diff --git a/third_party/move/move-vm/integration-tests/src/tests/instantiation_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/instantiation_tests.rs index ef24344f33ae2..bab392eb037be 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/instantiation_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/instantiation_tests.rs @@ -14,7 +14,7 @@ use move_core_types::{ language_storage::{StructTag, TypeTag}, vm_status::StatusCode, }; -use move_vm_runtime::{config::VMConfig, move_vm::MoveVM}; +use move_vm_runtime::{config::VMConfig, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; @@ -118,7 +118,7 @@ fn instantiation_err() { cm.serialize(&mut mod_bytes).unwrap(); session - .publish_module(mod_bytes, addr, &mut UnmeteredGasMeter) + .publish_module(mod_bytes, addr, &mut UnmeteredGasMeter, &DummyCodeStorage) .expect("Module must publish"); let mut ty_arg = TypeTag::U128; @@ -131,7 +131,7 @@ fn instantiation_err() { })); } - let res = session.load_function(&cm.self_id(), ident_str!("f"), &[ty_arg]); + let res = session.load_function(&DummyCodeStorage, &cm.self_id(), ident_str!("f"), &[ty_arg]); assert!( res.is_err(), "Instantiation must fail at load time when converting from type tag to type " diff --git a/third_party/move/move-vm/integration-tests/src/tests/invariant_violation_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/invariant_violation_tests.rs index 4eed9200011dd..961d0e5209628 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/invariant_violation_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/invariant_violation_tests.rs @@ -6,7 +6,7 @@ use move_binary_format::file_format::{ SignatureToken::*, }; use move_core_types::vm_status::StatusCode; -use move_vm_runtime::{module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; @@ -86,6 +86,8 @@ fn merge_borrow_states_infinite_loop() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .unwrap_err(); diff --git a/third_party/move/move-vm/integration-tests/src/tests/leak_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/leak_tests.rs index bb791af7b8d71..2c375454bf513 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/leak_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/leak_tests.rs @@ -4,7 +4,7 @@ use move_binary_format::file_format::{ Bytecode::*, CodeUnit, CompiledScript, Signature, SignatureIndex, SignatureToken::*, }; -use move_vm_runtime::{module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; @@ -61,6 +61,8 @@ fn leak_with_abort() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ); } diff --git a/third_party/move/move-vm/integration-tests/src/tests/loader_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/loader_tests.rs index f9d7b95439727..3e67df74d664b 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/loader_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/loader_tests.rs @@ -16,7 +16,7 @@ use move_core_types::{ identifier::{IdentStr, Identifier}, language_storage::ModuleId, }; -use move_vm_runtime::{config::VMConfig, module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{config::VMConfig, module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; use std::{path::PathBuf, sync::Arc, thread}; @@ -92,7 +92,12 @@ impl Adapter { .serialize(&mut binary) .unwrap_or_else(|_| panic!("failure in module serialization: {:#?}", module)); session - .publish_module(binary, WORKING_ACCOUNT, &mut UnmeteredGasMeter) + .publish_module( + binary, + WORKING_ACCOUNT, + &mut UnmeteredGasMeter, + &DummyCodeStorage, + ) .unwrap_or_else(|_| panic!("failure publishing module: {:#?}", module)); } let changeset = session.finish().expect("failure getting write set"); @@ -110,7 +115,12 @@ impl Adapter { .serialize(&mut binary) .unwrap_or_else(|_| panic!("failure in module serialization: {:#?}", module)); session - .publish_module(binary, WORKING_ACCOUNT, &mut UnmeteredGasMeter) + .publish_module( + binary, + WORKING_ACCOUNT, + &mut UnmeteredGasMeter, + &DummyCodeStorage, + ) .expect_err("publishing must fail"); } } @@ -138,6 +148,7 @@ impl Adapter { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_or_else(|_| { panic!("Failure executing {:?}::{:?}", module_id, name) @@ -161,6 +172,7 @@ impl Adapter { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_or_else(|_| panic!("Failure executing {:?}::{:?}", module, name)); } @@ -230,6 +242,8 @@ fn load_phantom_module() { let module_id = module.self_id(); adapter.publish_modules(vec![module]); + + #[allow(deprecated)] adapter.vm.load_module(&module_id, &adapter.store).unwrap(); } @@ -264,6 +278,8 @@ fn load_with_extra_ability() { let module_id = module.self_id(); adapter.publish_modules(vec![module]); + + #[allow(deprecated)] adapter.vm.load_module(&module_id, &adapter.store).unwrap(); } diff --git a/third_party/move/move-vm/integration-tests/src/tests/mutated_accounts_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/mutated_accounts_tests.rs index 8f390bc373579..f05b4f9273d9a 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/mutated_accounts_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/mutated_accounts_tests.rs @@ -9,7 +9,7 @@ use move_core_types::{ language_storage::ModuleId, value::{serialize_values, MoveValue}, }; -use move_vm_runtime::{module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; @@ -60,13 +60,14 @@ fn mutated_accounts() { serialize_values(&vec![MoveValue::Signer(account1)]), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); // The resource was published to "account1" and the sender's account // (TEST_ADDR) is assumed to be mutated as well (e.g., in a subsequent // transaction epilogue). - assert_eq!(sess.num_mutated_accounts(&TEST_ADDR), 2); + assert_eq!(sess.num_mutated_resources(&TEST_ADDR), 2); sess.execute_function_bypass_visibility( &module_id, @@ -75,10 +76,11 @@ fn mutated_accounts() { serialize_values(&vec![MoveValue::Address(account1)]), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); - assert_eq!(sess.num_mutated_accounts(&TEST_ADDR), 2); + assert_eq!(sess.num_mutated_resources(&TEST_ADDR), 2); sess.execute_function_bypass_visibility( &module_id, @@ -87,9 +89,10 @@ fn mutated_accounts() { serialize_values(&vec![MoveValue::Address(account1)]), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); - assert_eq!(sess.num_mutated_accounts(&TEST_ADDR), 2); + assert_eq!(sess.num_mutated_resources(&TEST_ADDR), 2); let changes = sess.finish().unwrap(); storage.apply(changes).unwrap(); @@ -102,9 +105,10 @@ fn mutated_accounts() { serialize_values(&vec![MoveValue::Address(account1)]), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); // Only the sender's account (TEST_ADDR) should have been modified. - assert_eq!(sess.num_mutated_accounts(&TEST_ADDR), 1); + assert_eq!(sess.num_mutated_resources(&TEST_ADDR), 1); } diff --git a/third_party/move/move-vm/integration-tests/src/tests/native_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/native_tests.rs index 70c8c02ca4db7..799169c648267 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/native_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/native_tests.rs @@ -9,6 +9,7 @@ use move_core_types::{ }; use move_vm_runtime::{ config::VMConfig, module_traversal::*, move_vm::MoveVM, native_functions::NativeFunction, + DummyCodeStorage, }; use move_vm_test_utils::InMemoryStorage; use move_vm_types::{gas::UnmeteredGasMeter, natives::function::NativeResult}; @@ -73,11 +74,21 @@ fn test_publish_module_with_nested_loops() { }); let mut sess = vm.new_session(&storage); - sess.publish_module(m_blob.clone(), TEST_ADDR, &mut UnmeteredGasMeter) - .unwrap(); + sess.publish_module( + m_blob.clone(), + TEST_ADDR, + &mut UnmeteredGasMeter, + &DummyCodeStorage, + ) + .unwrap(); let func = sess - .load_function(&m.self_id(), &Identifier::new("foo").unwrap(), &[]) + .load_function( + &DummyCodeStorage, + &m.self_id(), + &Identifier::new("foo").unwrap(), + &[], + ) .unwrap(); let err1 = sess .execute_entry_function( @@ -85,13 +96,19 @@ fn test_publish_module_with_nested_loops() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err(); assert!(err1.exec_state().unwrap().stack_trace().is_empty()); let func = sess - .load_function(&m.self_id(), &Identifier::new("foo2").unwrap(), &[]) + .load_function( + &DummyCodeStorage, + &m.self_id(), + &Identifier::new("foo2").unwrap(), + &[], + ) .unwrap(); let err2 = sess .execute_entry_function( @@ -99,6 +116,7 @@ fn test_publish_module_with_nested_loops() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err(); diff --git a/third_party/move/move-vm/integration-tests/src/tests/nested_loop_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/nested_loop_tests.rs index 92f1291964aa3..cd7370c42d255 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/nested_loop_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/nested_loop_tests.rs @@ -4,7 +4,7 @@ use crate::compiler::{as_module, as_script, compile_units}; use move_bytecode_verifier::VerifierConfig; use move_core_types::account_address::AccountAddress; -use move_vm_runtime::{config::VMConfig, module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{config::VMConfig, module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; @@ -53,8 +53,13 @@ fn test_publish_module_with_nested_loops() { ); let mut sess = vm.new_session(&storage); - sess.publish_module(m_blob.clone(), TEST_ADDR, &mut UnmeteredGasMeter) - .unwrap(); + sess.publish_module( + m_blob.clone(), + TEST_ADDR, + &mut UnmeteredGasMeter, + &DummyCodeStorage, + ) + .unwrap(); } // Should fail with max_loop_depth = 1 @@ -75,7 +80,7 @@ fn test_publish_module_with_nested_loops() { ); let mut sess = vm.new_session(&storage); - sess.publish_module(m_blob, TEST_ADDR, &mut UnmeteredGasMeter) + sess.publish_module(m_blob, TEST_ADDR, &mut UnmeteredGasMeter, &DummyCodeStorage) .unwrap_err(); } } @@ -131,6 +136,8 @@ fn test_run_script_with_nested_loops() { args, &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .unwrap(); } @@ -160,6 +167,8 @@ fn test_run_script_with_nested_loops() { args, &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .unwrap_err(); } diff --git a/third_party/move/move-vm/integration-tests/src/tests/regression_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/regression_tests.rs index 94b0f103d8c51..cf16193d89bb3 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/regression_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/regression_tests.rs @@ -10,7 +10,7 @@ use move_core_types::{ language_storage::{StructTag, TypeTag}, vm_status::StatusCode, }; -use move_vm_runtime::{config::VMConfig, module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{config::VMConfig, module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; use std::time::Instant; @@ -138,6 +138,8 @@ fn script_large_ty() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .unwrap_err(); diff --git a/third_party/move/move-vm/integration-tests/src/tests/return_value_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/return_value_tests.rs index b064de48e555d..5963c5abff814 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/return_value_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/return_value_tests.rs @@ -10,7 +10,9 @@ use move_core_types::{ language_storage::{ModuleId, TypeTag}, value::{MoveTypeLayout, MoveValue}, }; -use move_vm_runtime::{module_traversal::*, move_vm::MoveVM, session::SerializedReturnValues}; +use move_vm_runtime::{ + module_traversal::*, move_vm::MoveVM, session::SerializedReturnValues, DummyCodeStorage, +}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; @@ -71,6 +73,7 @@ fn run( args, &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, )?; Ok(return_values diff --git a/third_party/move/move-vm/integration-tests/src/tests/runtime_reentrancy_check_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/runtime_reentrancy_check_tests.rs index bc998a6b87a65..ed6aee5ea8d7e 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/runtime_reentrancy_check_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/runtime_reentrancy_check_tests.rs @@ -8,7 +8,9 @@ use move_core_types::{ account_address::AccountAddress, gas_algebra::GasQuantity, identifier::Identifier, language_storage::ModuleId, vm_status::StatusCode, }; -use move_vm_runtime::{module_traversal::*, move_vm::MoveVM, native_functions::NativeFunction}; +use move_vm_runtime::{ + module_traversal::*, move_vm::MoveVM, native_functions::NativeFunction, DummyCodeStorage, +}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::{gas::UnmeteredGasMeter, natives::function::NativeResult}; use smallvec::SmallVec; @@ -170,7 +172,8 @@ fn runtime_reentrancy_check() { vec![], args.clone(), &mut UnmeteredGasMeter, - &mut TraversalContext::new(&traversal_storage) + &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err() .major_status(), @@ -189,6 +192,7 @@ fn runtime_reentrancy_check() { args.clone(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap(); @@ -202,7 +206,8 @@ fn runtime_reentrancy_check() { vec![], args, &mut UnmeteredGasMeter, - &mut TraversalContext::new(&traversal_storage) + &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .unwrap_err() .major_status(), diff --git a/third_party/move/move-vm/integration-tests/src/tests/vm_arguments_tests.rs b/third_party/move/move-vm/integration-tests/src/tests/vm_arguments_tests.rs index 0bfcd2a96fb01..e21b19a0da813 100644 --- a/third_party/move/move-vm/integration-tests/src/tests/vm_arguments_tests.rs +++ b/third_party/move/move-vm/integration-tests/src/tests/vm_arguments_tests.rs @@ -21,7 +21,7 @@ use move_core_types::{ value::{serialize_values, MoveValue}, vm_status::{StatusCode, StatusType}, }; -use move_vm_runtime::{module_traversal::*, move_vm::MoveVM}; +use move_vm_runtime::{module_traversal::*, move_vm::MoveVM, DummyCodeStorage}; use move_vm_test_utils::InMemoryStorage; use move_vm_types::gas::UnmeteredGasMeter; @@ -263,6 +263,8 @@ fn call_script_with_args_ty_args_signers( combine_signers_and_args(signers, non_signer_args), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, + &DummyCodeStorage, ) .map(|_| ()) } @@ -295,6 +297,7 @@ fn call_script_function_with_args_ty_args_signers( combine_signers_and_args(signers, non_signer_args), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, )?; Ok(()) } @@ -792,6 +795,7 @@ fn call_missing_item() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .err() .unwrap(); @@ -816,6 +820,7 @@ fn call_missing_item() { Vec::>::new(), &mut UnmeteredGasMeter, &mut TraversalContext::new(&traversal_storage), + &DummyCodeStorage, ) .err() .unwrap(); diff --git a/third_party/move/move-vm/runtime/src/data_cache.rs b/third_party/move/move-vm/runtime/src/data_cache.rs index c562de0594e80..4cf34021a9eae 100644 --- a/third_party/move/move-vm/runtime/src/data_cache.rs +++ b/third_party/move/move-vm/runtime/src/data_cache.rs @@ -5,6 +5,7 @@ use crate::{ loader::{Loader, ModuleStorageAdapter}, logging::expect_no_verification_errors, + ModuleStorage, }; use bytes::Bytes; use move_binary_format::{ @@ -50,7 +51,7 @@ impl AccountDataCache { } } -fn load_module_impl( +fn fetch_module_impl( remote: &dyn MoveResolver, account_map: &BTreeMap, module_id: &ModuleId, @@ -171,7 +172,7 @@ impl<'r> TransactionDataCache<'r> { Ok(change_set) } - pub(crate) fn num_mutated_accounts(&self, sender: &AccountAddress) -> u64 { + pub(crate) fn num_mutated_resources(&self, sender: &AccountAddress) -> u64 { // The sender's account will always be mutated. let mut total_mutated_accounts: u64 = 1; for (addr, entry) in self.account_map.iter() { @@ -200,6 +201,7 @@ impl<'r> TransactionDataCache<'r> { pub(crate) fn load_resource( &mut self, loader: &Loader, + module_storage: &dyn ModuleStorage, addr: AccountAddress, ty: &Type, module_store: &ModuleStorageAdapter, @@ -219,28 +221,48 @@ impl<'r> TransactionDataCache<'r> { }, }; // TODO(Gas): Shall we charge for this? - let (ty_layout, has_aggregator_lifting) = - loader.type_to_type_layout_with_identifier_mappings(ty, module_store)?; - - let module = module_store.module_at(&ty_tag.module_id()); - let metadata: &[Metadata] = match &module { - Some(module) => &module.module().metadata, - None => &[], - }; - - // If we need to process aggregator lifting, we pass type layout to remote. - // Remote, in turn ensures that all aggregator values are lifted if the resolved - // resource comes from storage. - let (data, bytes_loaded) = self.remote.get_resource_bytes_with_metadata_and_layout( - &addr, - &ty_tag, - metadata, - if has_aggregator_lifting { - Some(&ty_layout) - } else { - None + let (ty_layout, has_aggregator_lifting) = loader + .type_to_type_layout_with_identifier_mappings(ty, module_store, module_storage)?; + + let (data, bytes_loaded) = match loader { + Loader::V1(_) => { + let maybe_module = module_store.module_at(&ty_tag.module_id()); + let metadata: &[Metadata] = match &maybe_module { + Some(m) => &m.module().metadata, + None => &[], + }; + // If we need to process aggregator lifting, we pass type layout to remote. + // Remote, in turn ensures that all aggregator values are lifted if the resolved + // resource comes from storage. + self.remote.get_resource_bytes_with_metadata_and_layout( + &addr, + &ty_tag, + metadata, + if has_aggregator_lifting { + Some(&ty_layout) + } else { + None + }, + )? + }, + Loader::V2(_) => { + let metadata = module_storage + .fetch_module_metadata(&ty_tag.address, ty_tag.module.as_ident_str())?; + // If we need to process aggregator lifting, we pass type layout to remote. + // Remote, in turn ensures that all aggregator values are lifted if the resolved + // resource comes from storage. + self.remote.get_resource_bytes_with_metadata_and_layout( + &addr, + &ty_tag, + metadata, + if has_aggregator_lifting { + Some(&ty_layout) + } else { + None + }, + )? }, - )?; + }; load_res = Some(NumBytes::new(bytes_loaded as u64)); let gv = match data { @@ -277,8 +299,8 @@ impl<'r> TransactionDataCache<'r> { )) } - pub(crate) fn load_module(&self, module_id: &ModuleId) -> PartialVMResult { - load_module_impl(self.remote, &self.account_map, module_id) + pub(crate) fn fetch_module(&self, module_id: &ModuleId) -> PartialVMResult { + fetch_module_impl(self.remote, &self.account_map, module_id) } pub(crate) fn load_compiled_script_to_cache( @@ -317,7 +339,7 @@ impl<'r> TransactionDataCache<'r> { btree_map::Entry::Occupied(entry) => Ok(entry.get().clone()), btree_map::Entry::Vacant(entry) => { // bytes fetching, allow loading to fail if the flag is set - let bytes = match load_module_impl(self.remote, &self.account_map, entry.key()) + let bytes = match fetch_module_impl(self.remote, &self.account_map, entry.key()) .map_err(|err| err.finish(Location::Undefined)) { Ok(bytes) => bytes, diff --git a/third_party/move/move-vm/runtime/src/interpreter.rs b/third_party/move/move-vm/runtime/src/interpreter.rs index 2baff79cc43df..337892c215fd0 100644 --- a/third_party/move/move-vm/runtime/src/interpreter.rs +++ b/third_party/move/move-vm/runtime/src/interpreter.rs @@ -9,7 +9,7 @@ use crate::{ module_traversal::TraversalContext, native_extensions::NativeContextExtensions, native_functions::NativeContext, - trace, LoadedFunction, + trace, LoadedFunction, ModuleStorage, }; use fail::fail_point; use move_binary_format::{ @@ -89,6 +89,7 @@ impl Interpreter { args: Vec, data_store: &mut TransactionDataCache, module_store: &ModuleStorageAdapter, + module_storage: &impl ModuleStorage, gas_meter: &mut impl GasMeter, traversal_context: &mut TraversalContext, extensions: &mut NativeContextExtensions, @@ -105,6 +106,7 @@ impl Interpreter { loader, data_store, module_store, + module_storage, gas_meter, traversal_context, extensions, @@ -124,6 +126,7 @@ impl Interpreter { loader: &Loader, data_store: &mut TransactionDataCache, module_store: &ModuleStorageAdapter, + module_storage: &impl ModuleStorage, gas_meter: &mut impl GasMeter, traversal_context: &mut TraversalContext, extensions: &mut NativeContextExtensions, @@ -149,12 +152,13 @@ impl Interpreter { self.access_control .enter_function(¤t_frame, ¤t_frame.function) .map_err(|e| self.set_location(e))?; + loop { - let resolver = current_frame.resolver(loader, module_store); - let exit_code = - current_frame //self - .execute_code(&resolver, &mut self, data_store, module_store, gas_meter) - .map_err(|err| self.attach_state_if_invariant_violation(err, ¤t_frame))?; + let resolver = current_frame.resolver(loader, module_store, module_storage); + let exit_code = current_frame + .execute_code(&resolver, &mut self, data_store, gas_meter) + .map_err(|err| self.attach_state_if_invariant_violation(err, ¤t_frame))?; + match exit_code { ExitCode::Return => { let non_ref_vals = current_frame @@ -224,7 +228,6 @@ impl Interpreter { &mut current_frame, &resolver, data_store, - module_store, gas_meter, traversal_context, extensions, @@ -278,7 +281,6 @@ impl Interpreter { &mut current_frame, &resolver, data_store, - module_store, gas_meter, traversal_context, extensions, @@ -424,7 +426,6 @@ impl Interpreter { current_frame: &mut Frame, resolver: &Resolver, data_store: &mut TransactionDataCache, - module_store: &ModuleStorageAdapter, gas_meter: &mut impl GasMeter, traversal_context: &mut TraversalContext, extensions: &mut NativeContextExtensions, @@ -435,7 +436,6 @@ impl Interpreter { current_frame, resolver, data_store, - module_store, gas_meter, traversal_context, extensions, @@ -464,7 +464,6 @@ impl Interpreter { current_frame: &mut Frame, resolver: &Resolver, data_store: &mut TransactionDataCache, - module_store: &ModuleStorageAdapter, gas_meter: &mut impl GasMeter, traversal_context: &mut TraversalContext, extensions: &mut NativeContextExtensions, @@ -576,18 +575,22 @@ impl Interpreter { } => { gas_meter.charge_native_function(cost, Option::>::None)?; - // Load the module that contains this function regardless of the traversal context. - // - // This is just a precautionary step to make sure that caching status of the VM will not alter execution - // result in case framework code forgot to use LoadFunction result to load the modules into cache - // and charge properly. - resolver - .loader() - .load_module(&module_name, data_store, module_store) - .map_err(|_| { - PartialVMError::new(StatusCode::FUNCTION_RESOLUTION_FAILURE) - .with_message(format!("Module {} doesn't exist", module_name)) - })?; + // Note(George): when V2 loader fetches the function, the defining module is + // automatically loaded as well, and there is no need for preloading of a module + // into the cache like in V1 design. + if let Loader::V1(loader) = resolver.loader() { + // Load the module that contains this function regardless of the traversal context. + // + // This is just a precautionary step to make sure that caching status of the VM will not alter execution + // result in case framework code forgot to use LoadFunction result to load the modules into cache + // and charge properly. + loader + .load_module(&module_name, data_store, resolver.module_store()) + .map_err(|_| { + PartialVMError::new(StatusCode::FUNCTION_RESOLUTION_FAILURE) + .with_message(format!("Module {} doesn't exist", module_name)) + })?; + } let target_func = resolver.build_loaded_function_from_name_and_ty_args( &module_name, &func_name, @@ -603,9 +606,7 @@ impl Interpreter { )); } - if resolver.loader().vm_config().disallow_dispatch_for_native - && target_func.is_native() - { + if resolver.vm_config().disallow_dispatch_for_native && target_func.is_native() { return Err(PartialVMError::new(StatusCode::RUNTIME_DISPATCH_ERROR) .with_message("Invoking native function during dispatch".to_string())); } @@ -650,25 +651,32 @@ impl Interpreter { resolver .loader() .check_dependencies_and_charge_gas( - module_store, + resolver.module_store(), data_store, gas_meter, &mut traversal_context.visited, traversal_context.referenced_modules, [(arena_id.address(), arena_id.name())], + resolver.module_storage(), ) .map_err(|err| err .to_partial() .append_message_with_separator('.', format!("Failed to charge transitive dependency for {}. Does this module exists?", module_name) ))?; - resolver - .loader() - .load_module(&module_name, data_store, module_store) - .map_err(|_| { - PartialVMError::new(StatusCode::FUNCTION_RESOLUTION_FAILURE) - .with_message(format!("Module {} doesn't exist", module_name)) - })?; + + // Note(George): same as above, when V2 loader fetches the function, the module + // where it is defined automatically loaded from ModuleStorage as well. There is + // no resolution via ModuleStorageAdapter like in V1 design, and it will be soon + // removed. + if let Loader::V1(loader) = resolver.loader() { + loader + .load_module(&module_name, data_store, resolver.module_store()) + .map_err(|_| { + PartialVMError::new(StatusCode::FUNCTION_RESOLUTION_FAILURE) + .with_message(format!("Module {} doesn't exist", module_name)) + })?; + } current_frame.pc += 1; // advance past the Call instruction in the caller Ok(()) @@ -748,19 +756,27 @@ impl Interpreter { /// Loads a resource from the data store and return the number of bytes read from the storage. fn load_resource<'c>( - loader: &Loader, + resolver: &Resolver, data_store: &'c mut TransactionDataCache, - module_store: &'c ModuleStorageAdapter, gas_meter: &mut impl GasMeter, addr: AccountAddress, ty: &Type, ) -> PartialVMResult<&'c mut GlobalValue> { - match data_store.load_resource(loader, addr, ty, module_store) { + match data_store.load_resource( + resolver.loader(), + resolver.module_storage(), + addr, + ty, + resolver.module_store(), + ) { Ok((gv, load_res)) => { if let Some(bytes_loaded) = load_res { gas_meter.charge_load_resource( addr, - TypeWithLoader { ty, loader }, + TypeWithLoader { + ty, + loader: resolver.loader(), + }, gv.view(), bytes_loaded, )?; @@ -776,23 +792,24 @@ impl Interpreter { &mut self, is_mut: bool, is_generic: bool, - loader: &Loader, + resolver: &Resolver, data_store: &mut TransactionDataCache, - module_store: &ModuleStorageAdapter, gas_meter: &mut impl GasMeter, addr: AccountAddress, ty: &Type, ) -> PartialVMResult<()> { - let res = Self::load_resource(loader, data_store, module_store, gas_meter, addr, ty)? - .borrow_global(); + let res = Self::load_resource(resolver, data_store, gas_meter, addr, ty)?.borrow_global(); gas_meter.charge_borrow_global( is_mut, is_generic, - TypeWithLoader { ty, loader }, + TypeWithLoader { + ty, + loader: resolver.loader(), + }, res.is_ok(), )?; self.check_access( - loader, + resolver.loader(), if is_mut { AccessKind::Writes } else { @@ -824,7 +841,7 @@ impl Interpreter { ) }, }; - let struct_name = &*loader.name_cache.idx_to_identifier(struct_idx); + let struct_name = &*loader.get_struct_name(struct_idx); if let Some(access) = AccessInstance::new(kind, struct_name, instance, addr) { self.access_control.check_access(access)? } @@ -835,17 +852,23 @@ impl Interpreter { fn exists( &mut self, is_generic: bool, - loader: &Loader, + resolver: &Resolver, data_store: &mut TransactionDataCache, - module_store: &ModuleStorageAdapter, gas_meter: &mut impl GasMeter, addr: AccountAddress, ty: &Type, ) -> PartialVMResult<()> { - let gv = Self::load_resource(loader, data_store, module_store, gas_meter, addr, ty)?; + let gv = Self::load_resource(resolver, data_store, gas_meter, addr, ty)?; let exists = gv.exists()?; - gas_meter.charge_exists(is_generic, TypeWithLoader { ty, loader }, exists)?; - self.check_access(loader, AccessKind::Reads, ty, addr)?; + gas_meter.charge_exists( + is_generic, + TypeWithLoader { + ty, + loader: resolver.loader(), + }, + exists, + )?; + self.check_access(resolver.loader(), AccessKind::Reads, ty, addr)?; self.operand_stack.push(Value::bool(exists))?; Ok(()) } @@ -854,34 +877,40 @@ impl Interpreter { fn move_from( &mut self, is_generic: bool, - loader: &Loader, + resolver: &Resolver, data_store: &mut TransactionDataCache, - module_store: &ModuleStorageAdapter, gas_meter: &mut impl GasMeter, addr: AccountAddress, ty: &Type, ) -> PartialVMResult<()> { - let resource = - match Self::load_resource(loader, data_store, module_store, gas_meter, addr, ty)? - .move_from() - { - Ok(resource) => { - gas_meter.charge_move_from( - is_generic, - TypeWithLoader { ty, loader }, - Some(&resource), - )?; - self.check_access(loader, AccessKind::Writes, ty, addr)?; - resource - }, - Err(err) => { - let val: Option<&Value> = None; - gas_meter.charge_move_from(is_generic, TypeWithLoader { ty, loader }, val)?; - return Err( - err.with_message(format!("Failed to move resource from {:?}", addr)) - ); - }, - }; + let resource = match Self::load_resource(resolver, data_store, gas_meter, addr, ty)? + .move_from() + { + Ok(resource) => { + gas_meter.charge_move_from( + is_generic, + TypeWithLoader { + ty, + loader: resolver.loader(), + }, + Some(&resource), + )?; + self.check_access(resolver.loader(), AccessKind::Writes, ty, addr)?; + resource + }, + Err(err) => { + let val: Option<&Value> = None; + gas_meter.charge_move_from( + is_generic, + TypeWithLoader { + ty, + loader: resolver.loader(), + }, + val, + )?; + return Err(err.with_message(format!("Failed to move resource from {:?}", addr))); + }, + }; self.operand_stack.push(resource)?; Ok(()) } @@ -890,32 +919,37 @@ impl Interpreter { fn move_to( &mut self, is_generic: bool, - loader: &Loader, + resolver: &Resolver, data_store: &mut TransactionDataCache, - module_store: &ModuleStorageAdapter, gas_meter: &mut impl GasMeter, addr: AccountAddress, ty: &Type, resource: Value, ) -> PartialVMResult<()> { - let gv = Self::load_resource(loader, data_store, module_store, gas_meter, addr, ty)?; + let gv = Self::load_resource(resolver, data_store, gas_meter, addr, ty)?; // NOTE(Gas): To maintain backward compatibility, we need to charge gas after attempting // the move_to operation. match gv.move_to(resource) { Ok(()) => { gas_meter.charge_move_to( is_generic, - TypeWithLoader { ty, loader }, + TypeWithLoader { + ty, + loader: resolver.loader(), + }, gv.view().unwrap(), true, )?; - self.check_access(loader, AccessKind::Writes, ty, addr)?; + self.check_access(resolver.loader(), AccessKind::Writes, ty, addr)?; Ok(()) }, Err((err, resource)) => { gas_meter.charge_move_to( is_generic, - TypeWithLoader { ty, loader }, + TypeWithLoader { + ty, + loader: resolver.loader(), + }, &resource, false, )?; @@ -1269,7 +1303,7 @@ impl CallStack { fn check_depth_of_type(resolver: &Resolver, ty: &Type) -> PartialVMResult<()> { // Start at 1 since we always call this right before we add a new node to the value's depth. - let max_depth = match resolver.loader().vm_config().max_value_nest_depth { + let max_depth = match resolver.vm_config().max_value_nest_depth { Some(max_depth) => max_depth, None => return Ok(()), }; @@ -1312,9 +1346,11 @@ fn check_depth_of_type_impl( }, Type::Vector(ty) => check_depth_of_type_impl(resolver, ty, max_depth, check_depth!(1))?, Type::Struct { idx, .. } => { - let formula = resolver - .loader() - .calculate_depth_of_struct(*idx, resolver.module_store())?; + let formula = resolver.loader().calculate_depth_of_struct( + *idx, + resolver.module_store(), + resolver.module_storage(), + )?; check_depth!(formula.solve(&[])) }, // NB: substitution must be performed before calling this function @@ -1327,9 +1363,11 @@ fn check_depth_of_type_impl( check_depth_of_type_impl(resolver, ty, max_depth, check_depth!(0)) }) .collect::>>()?; - let formula = resolver - .loader() - .calculate_depth_of_struct(*idx, resolver.module_store())?; + let formula = resolver.loader().calculate_depth_of_struct( + *idx, + resolver.module_store(), + resolver.module_storage(), + )?; check_depth!(formula.solve(&ty_arg_depths)) }, Type::TyParam(_) => { @@ -1562,10 +1600,9 @@ impl Frame { resolver: &Resolver, interpreter: &mut Interpreter, data_store: &mut TransactionDataCache, - module_store: &ModuleStorageAdapter, gas_meter: &mut impl GasMeter, ) -> VMResult { - self.execute_code_impl(resolver, interpreter, data_store, module_store, gas_meter) + self.execute_code_impl(resolver, interpreter, data_store, gas_meter) .map_err(|e| { let e = if cfg!(feature = "testing") || cfg!(feature = "stacktrace") { e.with_exec_state(interpreter.get_internal_state()) @@ -2259,7 +2296,6 @@ impl Frame { resolver: &Resolver, interpreter: &mut Interpreter, data_store: &mut TransactionDataCache, - module_store: &ModuleStorageAdapter, gas_meter: &mut impl GasMeter, ) -> PartialVMResult { use SimpleInstruction as S; @@ -2407,7 +2443,7 @@ impl Frame { Bytecode::MoveLoc(idx) => { let local = self.locals.move_loc( *idx as usize, - resolver.loader().vm_config().check_invariant_in_swap_loc, + resolver.vm_config().check_invariant_in_swap_loc, )?; gas_meter.charge_move_loc(&local)?; @@ -2419,7 +2455,7 @@ impl Frame { self.locals.store_loc( *idx as usize, value_to_store, - resolver.loader().vm_config().check_invariant_in_swap_loc, + resolver.vm_config().check_invariant_in_swap_loc, )?; }, Bytecode::Call(idx) => { @@ -2891,14 +2927,7 @@ impl Frame { let addr = interpreter.operand_stack.pop_as::()?; let ty = resolver.get_struct_ty(*sd_idx); interpreter.borrow_global( - is_mut, - false, - resolver.loader(), - data_store, - module_store, - gas_meter, - addr, - &ty, + is_mut, false, resolver, data_store, gas_meter, addr, &ty, )?; }, Bytecode::MutBorrowGlobalGeneric(si_idx) @@ -2912,28 +2941,13 @@ impl Frame { )?; gas_meter.charge_create_ty(ty_count)?; interpreter.borrow_global( - is_mut, - true, - resolver.loader(), - data_store, - module_store, - gas_meter, - addr, - ty, + is_mut, true, resolver, data_store, gas_meter, addr, ty, )?; }, Bytecode::Exists(sd_idx) => { let addr = interpreter.operand_stack.pop_as::()?; let ty = resolver.get_struct_ty(*sd_idx); - interpreter.exists( - false, - resolver.loader(), - data_store, - module_store, - gas_meter, - addr, - &ty, - )?; + interpreter.exists(false, resolver, data_store, gas_meter, addr, &ty)?; }, Bytecode::ExistsGeneric(si_idx) => { let addr = interpreter.operand_stack.pop_as::()?; @@ -2943,28 +2957,12 @@ impl Frame { self.function.ty_args(), )?; gas_meter.charge_create_ty(ty_count)?; - interpreter.exists( - true, - resolver.loader(), - data_store, - module_store, - gas_meter, - addr, - ty, - )?; + interpreter.exists(true, resolver, data_store, gas_meter, addr, ty)?; }, Bytecode::MoveFrom(sd_idx) => { let addr = interpreter.operand_stack.pop_as::()?; let ty = resolver.get_struct_ty(*sd_idx); - interpreter.move_from( - false, - resolver.loader(), - data_store, - module_store, - gas_meter, - addr, - &ty, - )?; + interpreter.move_from(false, resolver, data_store, gas_meter, addr, &ty)?; }, Bytecode::MoveFromGeneric(si_idx) => { let addr = interpreter.operand_stack.pop_as::()?; @@ -2974,15 +2972,7 @@ impl Frame { self.function.ty_args(), )?; gas_meter.charge_create_ty(ty_count)?; - interpreter.move_from( - true, - resolver.loader(), - data_store, - module_store, - gas_meter, - addr, - ty, - )?; + interpreter.move_from(true, resolver, data_store, gas_meter, addr, ty)?; }, Bytecode::MoveTo(sd_idx) => { let resource = interpreter.operand_stack.pop()?; @@ -2993,16 +2983,8 @@ impl Frame { .read_ref()? .value_as::()?; let ty = resolver.get_struct_ty(*sd_idx); - interpreter.move_to( - false, - resolver.loader(), - data_store, - module_store, - gas_meter, - addr, - &ty, - resource, - )?; + interpreter + .move_to(false, resolver, data_store, gas_meter, addr, &ty, resource)?; }, Bytecode::MoveToGeneric(si_idx) => { let resource = interpreter.operand_stack.pop()?; @@ -3018,16 +3000,8 @@ impl Frame { self.function.ty_args(), )?; gas_meter.charge_create_ty(ty_count)?; - interpreter.move_to( - true, - resolver.loader(), - data_store, - module_store, - gas_meter, - addr, - ty, - resource, - )?; + interpreter + .move_to(true, resolver, data_store, gas_meter, addr, ty, resource)?; }, Bytecode::FreezeRef => { gas_meter.charge_simple_instr(S::FreezeRef)?; @@ -3191,8 +3165,10 @@ impl Frame { &self, loader: &'a Loader, module_store: &'a ModuleStorageAdapter, + module_storage: &'a impl ModuleStorage, ) -> Resolver<'a> { - self.function.get_resolver(loader, module_store) + self.function + .get_resolver(loader, module_store, module_storage) } fn location(&self) -> Location { diff --git a/third_party/move/move-vm/runtime/src/lib.rs b/third_party/move/move-vm/runtime/src/lib.rs index 770862ddf2fbb..b328e951bea17 100644 --- a/third_party/move/move-vm/runtime/src/lib.rs +++ b/third_party/move/move-vm/runtime/src/lib.rs @@ -30,5 +30,9 @@ pub mod module_traversal; mod debug; mod access_control; +mod storage; pub use loader::LoadedFunction; +pub use storage::{ + dummy::DummyCodeStorage, module_storage::ModuleStorage, script_storage::ScriptStorage, +}; diff --git a/third_party/move/move-vm/runtime/src/loader/function.rs b/third_party/move/move-vm/runtime/src/loader/function.rs index 8ce692e9d87bf..a9792d60661a1 100644 --- a/third_party/move/move-vm/runtime/src/loader/function.rs +++ b/third_party/move/move-vm/runtime/src/loader/function.rs @@ -8,6 +8,7 @@ use crate::{ Resolver, Script, }, native_functions::{NativeFunction, NativeFunctions, UnboxedNativeFunction}, + ModuleStorage, }; use move_binary_format::{ access::ModuleAccess, @@ -144,13 +145,14 @@ impl LoadedFunction { &self, loader: &'a Loader, module_store: &'a ModuleStorageAdapter, + module_storage: &'a impl ModuleStorage, ) -> Resolver<'a> { match &self.owner { LoadedFunctionOwner::Module(module) => { - Resolver::for_module(loader, module_store, module.clone()) + Resolver::for_module(loader, module_store, module_storage, module.clone()) }, LoadedFunctionOwner::Script(script) => { - Resolver::for_script(loader, module_store, script.clone()) + Resolver::for_script(loader, module_store, module_storage, script.clone()) }, } } diff --git a/third_party/move/move-vm/runtime/src/loader/mod.rs b/third_party/move/move-vm/runtime/src/loader/mod.rs index f785ec41b0c96..9555682220c2a 100644 --- a/third_party/move/move-vm/runtime/src/loader/mod.rs +++ b/third_party/move/move-vm/runtime/src/loader/mod.rs @@ -5,6 +5,7 @@ use crate::{ config::VMConfig, data_cache::TransactionDataCache, logging::expect_no_verification_errors, module_traversal::TraversalContext, native_functions::NativeFunctions, + storage::module_storage::ModuleStorage as ModuleStorageV2, }; use hashbrown::Equivalent; use lazy_static::lazy_static; @@ -35,8 +36,7 @@ use move_vm_types::{ AbilityInfo, DepthFormula, StructIdentifier, StructNameIndex, StructType, Type, }, }; -use parking_lot::{MappedRwLockReadGuard, Mutex, RwLock, RwLockReadGuard}; -use sha3::{Digest, Sha3_256}; +use parking_lot::{MappedRwLockReadGuard, Mutex, RwLock}; use std::{ collections::{btree_map, BTreeMap, BTreeSet}, hash::Hash, @@ -50,7 +50,16 @@ mod modules; mod script; mod type_loader; -use crate::loader::modules::{StructVariantInfo, VariantFieldInfo}; +use crate::{ + loader::modules::{StructVariantInfo, VariantFieldInfo}, + storage::{ + dummy::DummyVerifier, + loader::LoaderV2, + script_storage::{script_hash, ScriptStorage}, + struct_name_index_map::StructNameIndexMap, + struct_type_storage::LoaderV1StructTypeStorage, + }, +}; pub use function::LoadedFunction; pub(crate) use function::{Function, FunctionHandle, FunctionInstantiation, LoadedFunctionOwner}; pub(crate) use modules::{Module, ModuleCache, ModuleStorage, ModuleStorageAdapter}; @@ -114,61 +123,281 @@ lazy_static! { Mutex::new(lru::LruCache::new(VERIFIED_CACHE_SIZE)); } -pub(crate) struct StructNameCache { - data: RwLock<( - BTreeMap, - Vec, - )>, +// +// Loader +// + +#[derive(Clone)] +pub(crate) enum Loader { + V1(LoaderV1), + #[allow(dead_code)] + V2(LoaderV2), } -impl Clone for StructNameCache { - fn clone(&self) -> Self { - let inner = self.data.read(); - Self { - data: RwLock::new((inner.0.clone(), inner.1.clone())), +macro_rules! versioned_loader_getter { + ($getter:ident, $return_ty:ty) => { + pub(crate) fn $getter(&self) -> &$return_ty { + match self { + Self::V1(loader) => loader.$getter(), + Self::V2(loader) => loader.$getter(), + } } - } + }; } -impl StructNameCache { - pub(crate) fn new() -> Self { - Self { - data: RwLock::new((BTreeMap::new(), vec![])), +impl Loader { + versioned_loader_getter!(vm_config, VMConfig); + + versioned_loader_getter!(struct_name_index_map, StructNameIndexMap); + + versioned_loader_getter!(ty_builder, TypeBuilder); + + versioned_loader_getter!(ty_cache, RwLock); + + pub(crate) fn new(natives: NativeFunctions, vm_config: VMConfig) -> Self { + Self::V1(LoaderV1 { + scripts: RwLock::new(ScriptCache::new()), + type_cache: RwLock::new(TypeCache::empty()), + struct_name_index_map: StructNameIndexMap::empty(), + natives, + invalidated: RwLock::new(false), + module_cache_hits: RwLock::new(BTreeSet::new()), + vm_config, + }) + } + + /// Flush this cache if it is marked as invalidated. + pub(crate) fn flush_if_invalidated(&self) { + if let Self::V1(loader) = self { + let mut invalidated = loader.invalidated.write(); + if *invalidated { + *loader.scripts.write() = ScriptCache::new(); + *loader.type_cache.write() = TypeCache::empty(); + *invalidated = false; + } } } - pub(crate) fn insert_or_get(&self, name: StructIdentifier) -> StructNameIndex { - if let Some(idx) = self.data.read().0.get(&name) { - return *idx; + /// Mark this cache as invalidated. + pub(crate) fn mark_as_invalid(&self) { + if let Self::V1(loader) = self { + *loader.invalidated.write() = true; } - let mut inner_data = self.data.write(); - let idx = StructNameIndex(inner_data.1.len()); - inner_data.0.insert(name.clone(), idx); - inner_data.1.push(name); - idx } - pub(crate) fn idx_to_identifier( + /// Check whether this cache is invalidated. + pub(crate) fn is_invalidated(&self) -> bool { + matches!(self, Self::V1(loader) if *loader.invalidated.read()) + } + + pub(crate) fn check_script_dependencies_and_check_gas( &self, - idx: StructNameIndex, + module_store: &ModuleStorageAdapter, + data_store: &mut TransactionDataCache, + gas_meter: &mut impl GasMeter, + traversal_context: &mut TraversalContext, + script_blob: &[u8], + module_storage: &impl ModuleStorageV2, + script_storage: &impl ScriptStorage, + ) -> VMResult<()> { + match self { + Self::V1(loader) => loader.check_script_dependencies_and_check_gas( + module_store, + data_store, + gas_meter, + traversal_context, + script_blob, + ), + Self::V2(loader) => loader + .check_script_dependencies_and_check_gas( + module_storage, + script_storage, + gas_meter, + traversal_context, + script_blob, + ) + .map_err(|e| e.finish(Location::Undefined)), + } + } + + pub(crate) fn check_dependencies_and_charge_gas<'a, I>( + &self, + module_store: &ModuleStorageAdapter, + data_store: &mut TransactionDataCache, + gas_meter: &mut impl GasMeter, + visited: &mut BTreeMap<(&'a AccountAddress, &'a IdentStr), ()>, + referenced_modules: &'a Arena>, + ids: I, + module_storage: &dyn ModuleStorageV2, + ) -> VMResult<()> + where + I: IntoIterator, + I::IntoIter: DoubleEndedIterator, + { + match self { + Self::V1(loader) => loader.check_dependencies_and_charge_gas( + module_store, + data_store, + gas_meter, + visited, + referenced_modules, + ids, + ), + Self::V2(loader) => loader + .check_dependencies_and_charge_gas( + module_storage, + gas_meter, + visited, + referenced_modules, + ids, + ) + .map_err(|e| e.finish(Location::Undefined)), + } + } + + pub(crate) fn load_script( + &self, + script_blob: &[u8], + ty_args: &[TypeTag], + data_store: &mut TransactionDataCache, + module_store: &ModuleStorageAdapter, + module_storage: &impl ModuleStorageV2, + script_storage: &impl ScriptStorage, + ) -> VMResult { + match self { + Self::V1(loader) => loader.load_script(script_blob, ty_args, data_store, module_store), + Self::V2(loader) => loader + .load_script(module_storage, script_storage, script_blob, ty_args) + .map_err(|e| e.finish(Location::Undefined)), + } + } + + pub(crate) fn load_module( + &self, + id: &ModuleId, + data_store: &mut TransactionDataCache, + module_store: &ModuleStorageAdapter, + module_storage: &impl ModuleStorageV2, + ) -> VMResult> { + match self { + Loader::V1(loader) => loader.load_module(id, data_store, module_store), + Loader::V2(loader) => loader + .load_module(module_storage, id.address(), id.name()) + .map_err(|e| e.finish(Location::Undefined)), + } + } + + fn load_function_without_type_args( + &self, + module_id: &ModuleId, + function_name: &IdentStr, + data_store: &mut TransactionDataCache, + module_store: &ModuleStorageAdapter, + module_storage: &impl ModuleStorageV2, + ) -> VMResult<(Arc, Arc)> { + match self { + Loader::V1(loader) => { + // Need to load the module first, before resolving it and the function. + loader.load_module(module_id, data_store, module_store)?; + module_store + .resolve_module_and_function_by_name(module_id, function_name) + .map_err(|err| err.finish(Location::Undefined)) + }, + Loader::V2(loader) => loader + .load_function_without_ty_args( + module_storage, + module_id.address(), + module_id.name(), + function_name, + ) + .map_err(|e| e.finish(Location::Undefined)), + } + } + + pub(crate) fn verify_module_bundle_for_publication( + &self, + modules: &[CompiledModule], + data_store: &mut TransactionDataCache, + module_store: &ModuleStorageAdapter, + module_storage: &impl ModuleStorageV2, + ) -> VMResult<()> { + match self { + Self::V1(loader) => { + loader.verify_module_bundle_for_publication(modules, data_store, module_store) + }, + Self::V2(loader) => loader + .verify_modules_for_publication(module_storage, modules) + .map_err(|e| e.finish(Location::Undefined)), + } + } + + // + // Helpers for loading and verification + // + + pub(crate) fn load_type( + &self, + ty_tag: &TypeTag, + data_store: &mut TransactionDataCache, + module_store: &ModuleStorageAdapter, + module_storage: &impl ModuleStorageV2, + ) -> VMResult { + match self { + Self::V1(loader) => loader.load_type(ty_tag, data_store, module_store), + Self::V2(loader) => loader + .load_ty(module_storage, ty_tag) + .map_err(|e| e.finish(Location::Undefined)), + } + } + + // + // Internal helpers + // + + pub(crate) fn get_struct_name( + &self, + struct_idx: StructNameIndex, ) -> MappedRwLockReadGuard { - RwLockReadGuard::map(self.data.read(), |inner| &inner.1[idx.0]) + match self { + Self::V1(loader) => loader + .struct_name_index_map() + .idx_to_struct_name(struct_idx), + Self::V2(loader) => loader + .struct_name_index_map() + .idx_to_struct_name(struct_idx), + } } -} -// -// Loader -// + pub fn fetch_struct_ty_by_idx( + &self, + idx: StructNameIndex, + module_store: &ModuleStorageAdapter, + module_storage: &dyn ModuleStorageV2, + ) -> PartialVMResult> { + let struct_name = self.struct_name_index_map().idx_to_struct_name(idx); + match self { + Loader::V1(_) => { + module_store.get_struct_type_by_identifier(&struct_name.name, &struct_name.module) + }, + Loader::V2(loader) => loader.load_struct_ty( + module_storage, + struct_name.module.address(), + struct_name.module.name(), + struct_name.name.as_ident_str(), + ), + } + } +} // A Loader is responsible to load scripts and modules and holds the cache of all loaded // entities. Each cache is protected by a `RwLock`. Operation in the Loader must be thread safe // (operating on values on the stack) and when cache needs updating the mutex must be taken. // The `pub(crate)` API is what a Loader offers to the runtime. -pub(crate) struct Loader { +pub(crate) struct LoaderV1 { scripts: RwLock, type_cache: RwLock, natives: NativeFunctions, - pub(crate) name_cache: StructNameCache, + struct_name_index_map: StructNameIndexMap, // The below field supports a hack to workaround well-known issues with the // loader cache. This cache is not designed to support module upgrade or deletion. @@ -203,13 +432,13 @@ pub(crate) struct Loader { vm_config: VMConfig, } -impl Clone for Loader { +impl Clone for LoaderV1 { fn clone(&self) -> Self { Self { scripts: RwLock::new(self.scripts.read().clone()), type_cache: RwLock::new(self.type_cache.read().clone()), natives: self.natives.clone(), - name_cache: self.name_cache.clone(), + struct_name_index_map: self.struct_name_index_map.clone(), invalidated: RwLock::new(*self.invalidated.read()), module_cache_hits: RwLock::new(self.module_cache_hits.read().clone()), vm_config: self.vm_config.clone(), @@ -217,19 +446,7 @@ impl Clone for Loader { } } -impl Loader { - pub(crate) fn new(natives: NativeFunctions, vm_config: VMConfig) -> Self { - Self { - scripts: RwLock::new(ScriptCache::new()), - type_cache: RwLock::new(TypeCache::new()), - name_cache: StructNameCache::new(), - natives, - invalidated: RwLock::new(false), - module_cache_hits: RwLock::new(BTreeSet::new()), - vm_config, - } - } - +impl LoaderV1 { pub(crate) fn vm_config(&self) -> &VMConfig { &self.vm_config } @@ -238,24 +455,12 @@ impl Loader { &self.vm_config.ty_builder } - /// Flush this cache if it is marked as invalidated. - pub(crate) fn flush_if_invalidated(&self) { - let mut invalidated = self.invalidated.write(); - if *invalidated { - *self.scripts.write() = ScriptCache::new(); - *self.type_cache.write() = TypeCache::new(); - *invalidated = false; - } - } - - /// Mark this cache as invalidated. - pub(crate) fn mark_as_invalid(&self) { - *self.invalidated.write() = true; + pub(crate) fn ty_cache(&self) -> &RwLock { + &self.type_cache } - /// Check whether this cache is invalidated. - pub(crate) fn is_invalidated(&self) -> bool { - *self.invalidated.read() + pub(crate) fn struct_name_index_map(&self) -> &StructNameIndexMap { + &self.struct_name_index_map } // @@ -270,11 +475,8 @@ impl Loader { traversal_context: &mut TraversalContext, script_blob: &[u8], ) -> VMResult<()> { - let mut sha3_256 = Sha3_256::new(); - sha3_256.update(script_blob); - let hash_value: [u8; 32] = sha3_256.finalize().into(); - - let script = data_store.load_compiled_script_to_cache(script_blob, hash_value)?; + let script = + data_store.load_compiled_script_to_cache(script_blob, script_hash(script_blob))?; let script = traversal_context.referenced_scripts.alloc(script); // TODO(Gas): Should we charge dependency gas for the script itself? @@ -306,10 +508,7 @@ impl Loader { module_store: &ModuleStorageAdapter, ) -> VMResult { // Retrieve or load the script. - let mut sha3_256 = Sha3_256::new(); - sha3_256.update(script_blob); - let hash_value: [u8; 32] = sha3_256.finalize().into(); - + let hash_value = script_hash(script_blob); let mut scripts = self.scripts.write(); let script = match scripts.get(&hash_value) { Some(cached) => cached, @@ -320,7 +519,11 @@ impl Loader { data_store, module_store, )?; - let script = Script::new(ver_script, module_store, &self.name_cache)?; + + let struct_ty_storage = LoaderV1StructTypeStorage { module_store }; + let script = + Script::new(ver_script, &struct_ty_storage, &self.struct_name_index_map) + .map_err(|e| e.finish(Location::Script))?; scripts.insert(hash_value, script) }, }; @@ -353,7 +556,7 @@ impl Loader { fn deserialize_and_verify_script( &self, script: &[u8], - hash_value: [u8; 32], + hash_value: ScriptHash, data_store: &mut TransactionDataCache, module_store: &ModuleStorageAdapter, ) -> VMResult> { @@ -378,105 +581,9 @@ impl Loader { // // Module verification and loading // +} - // Loading verifies the module if it was never loaded. - fn load_function_without_type_args( - &self, - module_id: &ModuleId, - function_name: &IdentStr, - data_store: &mut TransactionDataCache, - module_store: &ModuleStorageAdapter, - ) -> VMResult<(Arc, Arc)> { - // Need to load the module first, before resolving it and the function. - self.load_module(module_id, data_store, module_store)?; - module_store - .resolve_module_and_function_by_name(module_id, function_name) - .map_err(|err| err.finish(Location::Undefined)) - } - - // Matches the actual returned type to the expected type, binding any type args to the - // necessary type as stored in the map. The expected type must be a concrete type (no TyParam). - // Returns true if a successful match is made. - fn match_return_type<'a>( - returned: &Type, - expected: &'a Type, - map: &mut BTreeMap, - ) -> bool { - match (returned, expected) { - // The important case, deduce the type params - (Type::TyParam(idx), _) => match map.entry(*idx) { - btree_map::Entry::Vacant(vacant_entry) => { - vacant_entry.insert(expected); - true - }, - btree_map::Entry::Occupied(occupied_entry) => *occupied_entry.get() == expected, - }, - // Recursive types we need to recurse the matching types - (Type::Reference(ret_inner), Type::Reference(expected_inner)) - | (Type::MutableReference(ret_inner), Type::MutableReference(expected_inner)) => { - Self::match_return_type(ret_inner, expected_inner, map) - }, - (Type::Vector(ret_inner), Type::Vector(expected_inner)) => { - Self::match_return_type(ret_inner, expected_inner, map) - }, - // Abilities should not contribute to the equality check as they just serve for caching computations. - // For structs the both need to be the same struct. - ( - Type::Struct { idx: ret_idx, .. }, - Type::Struct { - idx: expected_idx, .. - }, - ) => *ret_idx == *expected_idx, - // For struct instantiations we need to additionally match all type arguments - ( - Type::StructInstantiation { - idx: ret_idx, - ty_args: ret_fields, - .. - }, - Type::StructInstantiation { - idx: expected_idx, - ty_args: expected_fields, - .. - }, - ) => { - *ret_idx == *expected_idx - && ret_fields.len() == expected_fields.len() - && ret_fields - .iter() - .zip(expected_fields.iter()) - .all(|types| Self::match_return_type(types.0, types.1, map)) - }, - // For primitive types we need to assure the types match - (Type::U8, Type::U8) - | (Type::U16, Type::U16) - | (Type::U32, Type::U32) - | (Type::U64, Type::U64) - | (Type::U128, Type::U128) - | (Type::U256, Type::U256) - | (Type::Bool, Type::Bool) - | (Type::Address, Type::Address) - | (Type::Signer, Type::Signer) => true, - // Otherwise the types do not match and we can't match return type to the expected type. - // Note we don't use the _ pattern but spell out all cases, so that the compiler will - // bark when a case is missed upon future updates to the types. - (Type::U8, _) - | (Type::U16, _) - | (Type::U32, _) - | (Type::U64, _) - | (Type::U128, _) - | (Type::U256, _) - | (Type::Bool, _) - | (Type::Address, _) - | (Type::Signer, _) - | (Type::Struct { .. }, _) - | (Type::StructInstantiation { .. }, _) - | (Type::Vector(_), _) - | (Type::MutableReference(_), _) - | (Type::Reference(_), _) => false, - } - } - +impl Loader { // Loading verifies the module if it was never loaded. // Type parameters are inferred from the expected return type. Returns an error if it's not // possible to infer the type parameters or return type cannot be matched. @@ -488,12 +595,14 @@ impl Loader { expected_return_type: &Type, data_store: &mut TransactionDataCache, module_store: &ModuleStorageAdapter, + module_storage: &impl ModuleStorageV2, ) -> VMResult { let (module, function) = self.load_function_without_type_args( module_id, function_name, data_store, module_store, + module_storage, )?; if function.return_tys().len() != 1 { @@ -502,7 +611,7 @@ impl Loader { } let mut map = BTreeMap::new(); - if !Self::match_return_type(&function.return_tys()[0], expected_return_type, &mut map) { + if !match_return_type(&function.return_tys()[0], expected_return_type, &mut map) { // For functions that are marked constructor this should not happen. return Err( PartialVMError::new(StatusCode::INVALID_MAIN_FUNCTION_SIGNATURE) @@ -545,17 +654,19 @@ impl Loader { ty_args: &[TypeTag], data_store: &mut TransactionDataCache, module_store: &ModuleStorageAdapter, + module_storage: &impl ModuleStorageV2, ) -> VMResult { let (module, function) = self.load_function_without_type_args( module_id, function_name, data_store, module_store, + module_storage, )?; let ty_args = ty_args .iter() - .map(|ty_arg| self.load_type(ty_arg, data_store, module_store)) + .map(|ty_arg| self.load_type(ty_arg, data_store, module_store, module_storage)) .collect::>>() .map_err(|mut err| { // User provided type argument failed to load. Set extra sub status to distinguish from internal type loading error. @@ -574,7 +685,9 @@ impl Loader { function, }) } +} +impl LoaderV1 { // Entry point for module publishing (`MoveVM::publish_module_bundle`). // // All modules in the bundle to be published must be loadable. This function performs all @@ -715,21 +828,19 @@ impl Loader { } fn check_natives(&self, module: &CompiledModule) -> VMResult<()> { - fn check_natives_impl(_loader: &Loader, module: &CompiledModule) -> PartialVMResult<()> { - // TODO: fix check and error code if we leave something around for native structs. - // For now this generates the only error test cases care about... - for (idx, struct_def) in module.struct_defs().iter().enumerate() { - if struct_def.field_information == StructFieldInformation::Native { - return Err(verification_error( - StatusCode::MISSING_DEPENDENCY, - IndexKind::FunctionHandle, - idx as TableIndex, - )); - } + // TODO: fix check and error code if we leave something around for native structs. + // For now this generates the only error test cases care about... + for (idx, struct_def) in module.struct_defs().iter().enumerate() { + if struct_def.field_information == StructFieldInformation::Native { + return Err(verification_error( + StatusCode::MISSING_DEPENDENCY, + IndexKind::FunctionHandle, + idx as TableIndex, + ) + .finish(Location::Module(module.self_id()))); } - Ok(()) } - check_natives_impl(self, module).map_err(|e| e.finish(Location::Module(module.self_id()))) + Ok(()) } // @@ -944,8 +1055,13 @@ impl Loader { )?; // if linking goes well, insert the module to the code cache - let module_ref = - module_store.insert(&self.natives, id.clone(), size, module, &self.name_cache)?; + let module_ref = module_store.insert( + &self.natives, + id.clone(), + size, + module, + &self.struct_name_index_map, + )?; Ok(module_ref) } @@ -1109,12 +1225,15 @@ pub(crate) struct Resolver<'a> { loader: &'a Loader, module_store: &'a ModuleStorageAdapter, binary: BinaryType, + + module_storage: &'a dyn ModuleStorageV2, } impl<'a> Resolver<'a> { fn for_module( loader: &'a Loader, module_store: &'a ModuleStorageAdapter, + module_storage: &'a dyn ModuleStorageV2, module: Arc, ) -> Self { let binary = BinaryType::Module(module); @@ -1122,12 +1241,14 @@ impl<'a> Resolver<'a> { loader, binary, module_store, + module_storage, } } fn for_script( loader: &'a Loader, module_store: &'a ModuleStorageAdapter, + module_storage: &'a dyn ModuleStorageV2, script: Arc