diff --git a/Cargo.lock b/Cargo.lock index 7f1d46f46aa30..2dda0881fc8fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1781,7 +1781,7 @@ dependencies = [ "linregress", "log", "parity-scale-codec", - "paste 1.0.4", + "paste 1.0.5", "serde", "sp-api", "sp-io", @@ -1872,7 +1872,7 @@ dependencies = [ "once_cell", "parity-scale-codec", "parity-util-mem", - "paste 1.0.4", + "paste 1.0.5", "pretty_assertions 0.6.1", "serde", "smallvec 1.6.1", @@ -4901,7 +4901,7 @@ dependencies = [ "pallet-timestamp", "pallet-utility", "parity-scale-codec", - "paste 1.0.4", + "paste 1.0.5", "pretty_assertions 0.7.2", "pwasm-utils", "rand 0.8.4", @@ -5001,7 +5001,7 @@ dependencies = [ "pallet-balances", "parity-scale-codec", "parking_lot 0.11.1", - "paste 1.0.4", + "paste 1.0.5", "rand 0.7.3", "sp-arithmetic", "sp-core", @@ -5513,7 +5513,7 @@ dependencies = [ "pallet-timestamp", "parity-scale-codec", "parking_lot 0.11.1", - "paste 1.0.4", + "paste 1.0.5", "rand_chacha 0.2.2", "serde", "sp-application-crypto", @@ -5979,9 +5979,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d65c4d95931acda4498f675e332fcbdc9a06705cd07086c510e9b6009cd1c1" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" [[package]] name = "paste-impl" @@ -7158,6 +7158,7 @@ dependencies = [ "log", "names", "parity-scale-codec", + "paste 1.0.5", "rand 0.7.3", "regex", "rpassword", @@ -7536,7 +7537,7 @@ dependencies = [ "parity-scale-codec", "parity-wasm 0.42.2", "parking_lot 0.11.1", - "paste 1.0.4", + "paste 1.0.5", "regex", "sc-executor-common", "sc-executor-wasmi", @@ -8584,7 +8585,7 @@ dependencies = [ "approx", "num-complex", "num-traits", - "paste 1.0.4", + "paste 1.0.5", ] [[package]] @@ -9179,7 +9180,7 @@ dependencies = [ "log", "parity-scale-codec", "parity-util-mem", - "paste 1.0.4", + "paste 1.0.5", "rand 0.7.3", "serde", "serde_json", @@ -11094,7 +11095,7 @@ dependencies = [ "lazy_static", "libc", "log", - "paste 1.0.4", + "paste 1.0.5", "psm", "region", "rustc-demangle", diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index f7cf8db003c0e..857f3a81af1d1 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -97,7 +97,7 @@ pub fn from_block_number(n: u32) -> Header { } pub fn executor() -> NativeExecutor { - NativeExecutor::new(WasmExecutionMethod::Interpreted, None, 8) + NativeExecutor::new(WasmExecutionMethod::Interpreted, 8) } pub fn executor_call< @@ -117,6 +117,7 @@ pub fn executor_call< let heap_pages = t.storage(sp_core::storage::well_known_keys::HEAP_PAGES); let runtime_code = RuntimeCode { code_fetcher: &sp_core::traits::WrappedRuntimeCode(code.as_slice().into()), + context: sp_core::traits::CodeContext::Consensus, hash: sp_core::blake2_256(&code).to_vec(), heap_pages: heap_pages.and_then(|hp| Decode::decode(&mut &hp[..]).ok()), }; diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 6aaaab04b627e..28c65a8735417 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -41,7 +41,7 @@ use node_runtime::{ }; use sc_block_builder::BlockBuilderProvider; use sc_client_api::{ - execution_extensions::{ExecutionExtensions, ExecutionStrategies}, + execution_extensions::{ExecutionConfig, ExecutionConfigs, ExecutionExtensions}, BlockBackend, ExecutionStrategy, }; use sc_client_db::PruningMode; @@ -390,11 +390,11 @@ impl BenchDb { let backend = sc_service::new_db_backend(db_config).expect("Should not fail"); let client = sc_service::new_client( backend.clone(), - NativeExecutor::new(WasmExecutionMethod::Compiled, None, 8), + NativeExecutor::new(WasmExecutionMethod::Compiled, 8), &keyring.generate_genesis(), None, None, - ExecutionExtensions::new(profile.into_execution_strategies(), None, None), + ExecutionExtensions::new(profile.into_execution_configs(), None, None), Box::new(task_executor.clone()), None, None, @@ -588,7 +588,7 @@ impl BenchKeyring { } } -/// Profile for exetion strategies. +/// Profile for execution configurations. #[derive(Clone, Copy, Debug)] pub enum Profile { /// As native as possible. @@ -598,21 +598,23 @@ pub enum Profile { } impl Profile { - fn into_execution_strategies(self) -> ExecutionStrategies { + fn into_execution_configs(self) -> ExecutionConfigs { match self { - Profile::Wasm => ExecutionStrategies { - syncing: ExecutionStrategy::AlwaysWasm, - importing: ExecutionStrategy::AlwaysWasm, - block_construction: ExecutionStrategy::AlwaysWasm, - offchain_worker: ExecutionStrategy::AlwaysWasm, - other: ExecutionStrategy::AlwaysWasm, + Profile::Wasm => ExecutionConfigs { + syncing: ExecutionConfig::new_offchain(ExecutionStrategy::AlwaysWasm), + importing: ExecutionConfig::new_offchain(ExecutionStrategy::AlwaysWasm), + block_construction: ExecutionConfig::new_offchain(ExecutionStrategy::AlwaysWasm), + offchain_worker: ExecutionConfig::new_offchain(ExecutionStrategy::AlwaysWasm), + other: ExecutionConfig::new_offchain(ExecutionStrategy::AlwaysWasm), }, - Profile::Native => ExecutionStrategies { - syncing: ExecutionStrategy::NativeElseWasm, - importing: ExecutionStrategy::NativeElseWasm, - block_construction: ExecutionStrategy::NativeElseWasm, - offchain_worker: ExecutionStrategy::NativeElseWasm, - other: ExecutionStrategy::NativeElseWasm, + Profile::Native => ExecutionConfigs { + syncing: ExecutionConfig::new_offchain(ExecutionStrategy::NativeElseWasm), + importing: ExecutionConfig::new_offchain(ExecutionStrategy::NativeElseWasm), + block_construction: ExecutionConfig::new_offchain( + ExecutionStrategy::NativeElseWasm, + ), + offchain_worker: ExecutionConfig::new_offchain(ExecutionStrategy::NativeElseWasm), + other: ExecutionConfig::new_offchain(ExecutionStrategy::NativeElseWasm), }, } } diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index a19df74326068..7d08d9a6b5312 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -23,7 +23,7 @@ use sc_executor::{NativeVersion, RuntimeVersion}; use sp_core::NativeOrEncoded; use sp_externalities::Extensions; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; -use sp_state_machine::{ExecutionManager, ExecutionStrategy, OverlayedChanges, StorageProof}; +use sp_state_machine::{ExecutionConfig, ExecutionManager, OverlayedChanges, StorageProof}; use std::{cell::RefCell, panic::UnwindSafe, result}; use crate::execution_extensions::ExecutionExtensions; @@ -57,7 +57,7 @@ pub trait CallExecutor { id: &BlockId, method: &str, call_data: &[u8], - strategy: ExecutionStrategy, + config: ExecutionConfig, extensions: Option, ) -> Result, sp_blockchain::Error>; diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index ec44294b8a96c..eaf4620df186d 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -32,33 +32,33 @@ use sp_core::{ use sp_externalities::Extensions; use sp_keystore::{KeystoreExt, SyncCryptoStorePtr}; use sp_runtime::{generic::BlockId, traits}; -pub use sp_state_machine::ExecutionStrategy; use sp_state_machine::{DefaultHandler, ExecutionManager}; +pub use sp_state_machine::{ExecutionConfig, ExecutionStrategy}; use std::sync::{Arc, Weak}; -/// Execution strategies settings. +/// Execution configurations, per operation type. #[derive(Debug, Clone)] -pub struct ExecutionStrategies { - /// Execution strategy used when syncing. - pub syncing: ExecutionStrategy, - /// Execution strategy used when importing blocks. - pub importing: ExecutionStrategy, - /// Execution strategy used when constructing blocks. - pub block_construction: ExecutionStrategy, - /// Execution strategy used for offchain workers. - pub offchain_worker: ExecutionStrategy, - /// Execution strategy used in other cases. - pub other: ExecutionStrategy, +pub struct ExecutionConfigs { + /// Execution config used when syncing. + pub syncing: ExecutionConfig, + /// Execution config used when importing blocks. + pub importing: ExecutionConfig, + /// Execution config used when constructing blocks. + pub block_construction: ExecutionConfig, + /// Execution config used for offchain workers. + pub offchain_worker: ExecutionConfig, + /// Execution config used in other cases. + pub other: ExecutionConfig, } -impl Default for ExecutionStrategies { - fn default() -> ExecutionStrategies { - ExecutionStrategies { - syncing: ExecutionStrategy::NativeElseWasm, - importing: ExecutionStrategy::NativeElseWasm, - block_construction: ExecutionStrategy::AlwaysWasm, - offchain_worker: ExecutionStrategy::NativeWhenPossible, - other: ExecutionStrategy::NativeElseWasm, +impl Default for ExecutionConfigs { + fn default() -> ExecutionConfigs { + ExecutionConfigs { + syncing: ExecutionStrategy::NativeElseWasm.in_consensus(), + importing: ExecutionStrategy::NativeElseWasm.in_consensus(), + block_construction: ExecutionStrategy::AlwaysWasm.in_consensus(), + offchain_worker: ExecutionStrategy::NativeWhenPossible.in_offchain(), + other: ExecutionStrategy::NativeElseWasm.in_consensus(), } } } @@ -93,7 +93,7 @@ impl DbExternaliti /// and is responsible for producing a correct `Extensions` object. /// for each call, based on required `Capabilities`. pub struct ExecutionExtensions { - strategies: ExecutionStrategies, + configs: ExecutionConfigs, keystore: Option, offchain_db: Option>, // FIXME: these two are only RwLock because of https://github.com/paritytech/substrate/issues/4587 @@ -109,7 +109,7 @@ pub struct ExecutionExtensions { impl Default for ExecutionExtensions { fn default() -> Self { Self { - strategies: Default::default(), + configs: Default::default(), keystore: None, offchain_db: None, transaction_pool: RwLock::new(None), @@ -119,16 +119,16 @@ impl Default for ExecutionExtensions { } impl ExecutionExtensions { - /// Create new `ExecutionExtensions` given a `keystore` and `ExecutionStrategies`. + /// Create new `ExecutionExtensions` given a `keystore` and `ExecutionConfigs`. pub fn new( - strategies: ExecutionStrategies, + configs: ExecutionConfigs, keystore: Option, offchain_db: Option>, ) -> Self { let transaction_pool = RwLock::new(None); let extensions_factory = Box::new(()); Self { - strategies, + configs, keystore, offchain_db, extensions_factory: RwLock::new(extensions_factory), @@ -136,9 +136,9 @@ impl ExecutionExtensions { } } - /// Get a reference to the execution strategies. - pub fn strategies(&self) -> &ExecutionStrategies { - &self.strategies + /// Get a reference to the execution configs. + pub fn configs(&self) -> &ExecutionConfigs { + &self.configs } /// Set the new extensions_factory @@ -207,12 +207,12 @@ impl ExecutionExtensions { context: ExecutionContext, ) -> (ExecutionManager>, Extensions) { let manager = match context { - ExecutionContext::BlockConstruction => self.strategies.block_construction.get_manager(), - ExecutionContext::Syncing => self.strategies.syncing.get_manager(), - ExecutionContext::Importing => self.strategies.importing.get_manager(), + ExecutionContext::BlockConstruction => self.configs.block_construction.get_manager(), + ExecutionContext::Syncing => self.configs.syncing.get_manager(), + ExecutionContext::Importing => self.configs.importing.get_manager(), ExecutionContext::OffchainCall(Some((_, capabilities))) if capabilities.has_all() => - self.strategies.offchain_worker.get_manager(), - ExecutionContext::OffchainCall(_) => self.strategies.other.get_manager(), + self.configs.offchain_worker.get_manager(), + ExecutionContext::OffchainCall(_) => self.configs.other.get_manager(), }; (manager, self.extensions(at, context)) diff --git a/client/api/src/lib.rs b/client/api/src/lib.rs index 16935b1e846cf..69b0f6dc56ee3 100644 --- a/client/api/src/lib.rs +++ b/client/api/src/lib.rs @@ -39,7 +39,7 @@ pub use proof_provider::*; pub use sp_blockchain as blockchain; pub use sp_blockchain::HeaderBackend; -pub use sp_state_machine::{ExecutionStrategy, StorageProof}; +pub use sp_state_machine::{ExecutionConfig, ExecutionStrategy, StorageProof}; pub use sp_storage::{ChildInfo, PrefixedStorageKey, StorageData, StorageKey}; /// Usage Information Provider interface diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index f38686d228658..89ba1194c964d 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -43,6 +43,7 @@ sc-tracing = { version = "4.0.0-dev", path = "../tracing" } chrono = "0.4.10" serde = "1.0.126" thiserror = "1.0.21" +paste = "1.0.5" [target.'cfg(not(target_os = "unknown"))'.dependencies] rpassword = "5.0.0" diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs index 72741d7bea2bb..1d4ec77045334 100644 --- a/client/cli/src/arg_enums.rs +++ b/client/cli/src/arg_enums.rs @@ -18,6 +18,7 @@ // NOTE: we allow missing docs here because arg_enum! creates the function variants without doc #![allow(missing_docs)] +use sp_core::traits::CodeContext; use structopt::clap::arg_enum; /// How to execute Wasm runtime code. @@ -260,15 +261,24 @@ impl Into for SyncMode { } } -/// Default value for the `--execution-syncing` parameter. -pub const DEFAULT_EXECUTION_SYNCING: ExecutionStrategy = ExecutionStrategy::NativeElseWasm; -/// Default value for the `--execution-import-block` parameter. -pub const DEFAULT_EXECUTION_IMPORT_BLOCK: ExecutionStrategy = ExecutionStrategy::NativeElseWasm; -/// Default value for the `--execution-import-block` parameter when the node is a validator. -pub const DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR: ExecutionStrategy = ExecutionStrategy::Wasm; -/// Default value for the `--execution-block-construction` parameter. -pub const DEFAULT_EXECUTION_BLOCK_CONSTRUCTION: ExecutionStrategy = ExecutionStrategy::Wasm; -/// Default value for the `--execution-offchain-worker` parameter. -pub const DEFAULT_EXECUTION_OFFCHAIN_WORKER: ExecutionStrategy = ExecutionStrategy::Native; -/// Default value for the `--execution-other` parameter. -pub const DEFAULT_EXECUTION_OTHER: ExecutionStrategy = ExecutionStrategy::Native; +macro_rules! generate_config_const { + ($( $role:ident => $strategy:ident, $context:ident ),*) => { + paste::paste! { + $( + pub const []: ExecutionStrategy + = ExecutionStrategy::$strategy; + pub const []: CodeContext + = CodeContext::$context; + )* + } + }; +} + +generate_config_const!( + syncing => NativeElseWasm, Consensus, + import_block => NativeElseWasm, Consensus, + import_block_validator => Wasm, Consensus, + block_construction => Wasm, Consensus, + offchain_worker => Native, Offchain, + other => Native, Consensus +); diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index d586156410507..8baee21d376f8 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -24,7 +24,7 @@ use crate::{ }; use log::warn; use names::{Generator, Name}; -use sc_client_api::execution_extensions::ExecutionStrategies; +use sc_client_api::execution_extensions::ExecutionConfigs; use sc_service::{ config::{ BasePath, Configuration, DatabaseConfig, ExtTransport, KeystoreConfig, @@ -293,15 +293,11 @@ pub trait CliConfiguration: Sized { /// Get the execution strategies. /// /// By default this is retrieved from `ImportParams` if it is available. Otherwise its - /// `ExecutionStrategies::default()`. - fn execution_strategies( - &self, - is_dev: bool, - is_validator: bool, - ) -> Result { + /// `ExecutionConfigs::default()`. + fn execution_configs(&self, is_dev: bool, is_validator: bool) -> Result { Ok(self .import_params() - .map(|x| x.execution_strategies(is_dev, is_validator)) + .map(|x| x.execution_configs(is_dev, is_validator)) .unwrap_or_default()) } @@ -512,7 +508,7 @@ pub trait CliConfiguration: Sized { transaction_storage: self.database_transaction_storage()?, wasm_method: self.wasm_method()?, wasm_runtime_overrides: self.wasm_runtime_overrides(), - execution_strategies: self.execution_strategies(is_dev, is_validator)?, + execution_configs: self.execution_configs(is_dev, is_validator)?, rpc_http: self.rpc_http(DCV::rpc_http_listen_port())?, rpc_ws: self.rpc_ws(DCV::rpc_ws_listen_port())?, rpc_ipc: self.rpc_ipc()?, @@ -524,7 +520,6 @@ pub trait CliConfiguration: Sized { prometheus_config: self.prometheus_config(DCV::prometheus_listen_port())?, telemetry_endpoints, telemetry_external_transport: self.telemetry_external_transport()?, - default_heap_pages: self.default_heap_pages()?, offchain_worker: self.offchain_worker(&role)?, force_authoring: self.force_authoring()?, disable_grandpa: self.disable_grandpa()?, diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs index 9248e210eb662..5a4fbaadc2c0d 100644 --- a/client/cli/src/params/import_params.rs +++ b/client/cli/src/params/import_params.rs @@ -18,13 +18,16 @@ use crate::{ arg_enums::{ - ExecutionStrategy, WasmExecutionMethod, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, - DEFAULT_EXECUTION_IMPORT_BLOCK, DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR, - DEFAULT_EXECUTION_OFFCHAIN_WORKER, DEFAULT_EXECUTION_OTHER, DEFAULT_EXECUTION_SYNCING, + ExecutionStrategy, WasmExecutionMethod, DEFAULT_CODE_CONTEXT_BLOCK_CONSTRUCTION, + DEFAULT_CODE_CONTEXT_IMPORT_BLOCK, DEFAULT_CODE_CONTEXT_IMPORT_BLOCK_VALIDATOR, + DEFAULT_CODE_CONTEXT_OFFCHAIN_WORKER, DEFAULT_CODE_CONTEXT_OTHER, + DEFAULT_CODE_CONTEXT_SYNCING, DEFAULT_STRATEGY_BLOCK_CONSTRUCTION, + DEFAULT_STRATEGY_IMPORT_BLOCK, DEFAULT_STRATEGY_IMPORT_BLOCK_VALIDATOR, + DEFAULT_STRATEGY_OFFCHAIN_WORKER, DEFAULT_STRATEGY_OTHER, DEFAULT_STRATEGY_SYNCING, }, params::{DatabaseParams, PruningParams}, }; -use sc_client_api::execution_extensions::ExecutionStrategies; +use sc_client_api::execution_extensions::{ExecutionConfig, ExecutionConfigs}; use std::path::PathBuf; use structopt::StructOpt; @@ -96,32 +99,52 @@ impl ImportParams { } /// Get execution strategies for the parameters - pub fn execution_strategies(&self, is_dev: bool, is_validator: bool) -> ExecutionStrategies { + pub fn execution_configs(&self, is_dev: bool, is_validator: bool) -> ExecutionConfigs { let exec = &self.execution_strategies; - let exec_all_or = |strat: Option, default: ExecutionStrategy| { + let exec_all_or = |start: Option, default: ExecutionStrategy| { let default = if is_dev { ExecutionStrategy::Native } else { default }; - exec.execution.unwrap_or_else(|| strat.unwrap_or(default)).into() + exec.execution.unwrap_or_else(|| start.unwrap_or(default)).into() }; let default_execution_import_block = if is_validator { - DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR + DEFAULT_STRATEGY_IMPORT_BLOCK_VALIDATOR } else { - DEFAULT_EXECUTION_IMPORT_BLOCK + DEFAULT_STRATEGY_IMPORT_BLOCK + }; + let default_context_import_block = if is_validator { + DEFAULT_CODE_CONTEXT_IMPORT_BLOCK_VALIDATOR + } else { + DEFAULT_CODE_CONTEXT_IMPORT_BLOCK }; - ExecutionStrategies { - syncing: exec_all_or(exec.execution_syncing, DEFAULT_EXECUTION_SYNCING), - importing: exec_all_or(exec.execution_import_block, default_execution_import_block), - block_construction: exec_all_or( - exec.execution_block_construction, - DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, - ), - offchain_worker: exec_all_or( - exec.execution_offchain_worker, - DEFAULT_EXECUTION_OFFCHAIN_WORKER, - ), - other: exec_all_or(exec.execution_other, DEFAULT_EXECUTION_OTHER), + ExecutionConfigs { + syncing: ExecutionConfig { + strategy: exec_all_or(exec.execution_syncing, DEFAULT_STRATEGY_SYNCING), + context: DEFAULT_CODE_CONTEXT_SYNCING, + }, + importing: ExecutionConfig { + strategy: exec_all_or(exec.execution_import_block, default_execution_import_block), + context: default_context_import_block, + }, + block_construction: ExecutionConfig { + strategy: exec_all_or( + exec.execution_block_construction, + DEFAULT_STRATEGY_BLOCK_CONSTRUCTION, + ), + context: DEFAULT_CODE_CONTEXT_BLOCK_CONSTRUCTION, + }, + offchain_worker: ExecutionConfig { + strategy: exec_all_or( + exec.execution_offchain_worker, + DEFAULT_STRATEGY_OFFCHAIN_WORKER, + ), + context: DEFAULT_CODE_CONTEXT_OFFCHAIN_WORKER, + }, + other: ExecutionConfig { + strategy: exec_all_or(exec.execution_other, DEFAULT_STRATEGY_OTHER), + context: DEFAULT_CODE_CONTEXT_OTHER, + }, } } } diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index dabead4799dc8..b840322064a12 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -77,19 +77,15 @@ fn call_in_wasm( execution_method: WasmExecutionMethod, ext: &mut E, ) -> Result, String> { - let executor = crate::WasmExecutor::new( - execution_method, - Some(1024), - HostFunctions::host_functions(), - 8, - None, - ); + let executor = + crate::WasmExecutor::new(execution_method, HostFunctions::host_functions(), 8, None); executor.uncached_call( RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(), ext, true, function, call_data, + 1024, ) } @@ -427,13 +423,7 @@ test_wasm_execution!(should_trap_when_heap_exhausted); fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); - let executor = crate::WasmExecutor::new( - wasm_method, - Some(17), // `17` is the initial number of pages compiled into the binary. - HostFunctions::host_functions(), - 8, - None, - ); + let executor = crate::WasmExecutor::new(wasm_method, HostFunctions::host_functions(), 8, None); let err = executor .uncached_call( @@ -442,6 +432,8 @@ fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { true, "test_exhaust_heap", &[0], + // `17` is the initial number of pages compiled into the binary. + 17, ) .unwrap_err(); @@ -542,7 +534,6 @@ test_wasm_execution!(parallel_execution); fn parallel_execution(wasm_method: WasmExecutionMethod) { let executor = std::sync::Arc::new(crate::WasmExecutor::new( wasm_method, - Some(1024), HostFunctions::host_functions(), 8, None, @@ -561,6 +552,7 @@ fn parallel_execution(wasm_method: WasmExecutionMethod) { true, "test_twox_128", &[0], + 1024 ) .unwrap(), hex!("99e9d85137db46ef4bbea33613baafd5").to_vec().encode(), diff --git a/client/executor/src/lib.rs b/client/executor/src/lib.rs index f4b972a86f27a..674fc9e31d2d9 100644 --- a/client/executor/src/lib.rs +++ b/client/executor/src/lib.rs @@ -79,7 +79,6 @@ mod tests { let executor = WasmExecutor::new( WasmExecutionMethod::Interpreted, - Some(8), sp_io::SubstrateHostFunctions::host_functions(), 8, None, @@ -91,6 +90,7 @@ mod tests { true, "test_empty_return", &[], + 8, ) .unwrap(); assert_eq!(res, vec![0u8; 0]); diff --git a/client/executor/src/native_executor.rs b/client/executor/src/native_executor.rs index 8222e00b17615..cb5914ab35e4f 100644 --- a/client/executor/src/native_executor.rs +++ b/client/executor/src/native_executor.rs @@ -48,8 +48,14 @@ use sp_tasks::new_async_externalities; use sp_version::{NativeVersion, RuntimeVersion}; use sp_wasm_interface::{Function, HostFunctions}; -/// Default num of pages for the heap -const DEFAULT_HEAP_PAGES: u64 = 2048; +/// Default num of pages for the heap of the runtime, when used for consensus operations. +/// +/// 128mb per instance. +const DEFAULT_HEAP_PAGES_CONSENSUS: u64 = 2048; +/// Default num of pages for the heap of the runtime, when used for offchain operations. +/// +/// 256mb per instance. +const DEFAULT_HEAP_PAGES_OFFCHAIN: u64 = DEFAULT_HEAP_PAGES_CONSENSUS * 2; /// Set up the externalities and safe calling environment to execute runtime calls. /// @@ -97,8 +103,6 @@ pub trait NativeExecutionDispatch: Send + Sync { pub struct WasmExecutor { /// Method used to execute fallback Wasm code. method: WasmExecutionMethod, - /// The number of 64KB pages to allocate for Wasm execution. - default_heap_pages: u64, /// The host functions registered with this instance. host_functions: Arc>, /// WASM runtime cache. @@ -130,14 +134,12 @@ impl WasmExecutor { /// compiled execution method is used. pub fn new( method: WasmExecutionMethod, - default_heap_pages: Option, host_functions: Vec<&'static dyn Function>, max_runtime_instances: usize, cache_path: Option, ) -> Self { WasmExecutor { method, - default_heap_pages: default_heap_pages.unwrap_or(DEFAULT_HEAP_PAGES), host_functions: Arc::new(host_functions), cache: Arc::new(RuntimeCache::new(max_runtime_instances, cache_path.clone())), max_runtime_instances, @@ -152,6 +154,8 @@ impl WasmExecutor { /// prevent any poisoned state. Native runtime execution does not need to report back /// any `panic!`. /// + /// The number of heap pages to use is deduced from `runtime_code`'s context. + /// /// # Safety /// /// `runtime` and `ext` are given as `AssertUnwindSafe` to the closure. As described above, the @@ -173,11 +177,15 @@ impl WasmExecutor { AssertUnwindSafe<&mut dyn Externalities>, ) -> Result>, { + let default_heap_pages = match runtime_code.context { + sp_core::traits::CodeContext::Consensus => DEFAULT_HEAP_PAGES_CONSENSUS, + sp_core::traits::CodeContext::Offchain => DEFAULT_HEAP_PAGES_OFFCHAIN, + }; match self.cache.with_instance( runtime_code, ext, self.method, - self.default_heap_pages, + default_heap_pages, &*self.host_functions, allow_missing_host_functions, |module, instance, version, ext| { @@ -194,11 +202,14 @@ impl WasmExecutor { /// Perform a call into the given runtime. /// - /// The runtime is passed as a [`RuntimeBlob`]. The runtime will be isntantiated with the + /// The runtime is passed as a [`RuntimeBlob`]. The runtime will be instantiated with the /// parameters this `WasmExecutor` was initialized with. /// - /// In case of problems with during creation of the runtime or instantation, a `Err` is returned. - /// that describes the message. + /// In case of problems during creation of the runtime or instantiation, a `Err` is + /// returned. that describes the message. + /// + /// Unlike `with_instance`, the number of heap pages to use must be explicitly given to this + /// function. #[doc(hidden)] // We use this function for tests across multiple crates. pub fn uncached_call( &self, @@ -207,10 +218,11 @@ impl WasmExecutor { allow_missing_host_functions: bool, export_name: &str, call_data: &[u8], + heap_pages: u64, ) -> std::result::Result, String> { let module = crate::wasm_runtime::create_wasm_runtime_with_code( self.method, - self.default_heap_pages, + heap_pages, runtime_blob, self.host_functions.to_vec(), allow_missing_host_functions, @@ -265,12 +277,13 @@ impl sp_core::traits::ReadRuntimeVersion for WasmExecutor { true, "Core_version", &[], + DEFAULT_HEAP_PAGES_CONSENSUS, ) } } -/// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence -/// and dispatch to native code when possible, falling back on `WasmExecutor` when not. +/// A generic `CodeExecutor` implementation that uses a delegate `D` to determine wasm code +/// equivalence and dispatch to native code when possible, falling back on `WasmExecutor` when not. pub struct NativeExecutor { /// Dummy field to avoid the compiler complaining about us not using `D`. _dummy: std::marker::PhantomData, @@ -286,14 +299,9 @@ impl NativeExecutor { /// # Parameters /// /// `fallback_method` - Method used to execute fallback Wasm code. - /// /// `default_heap_pages` - Number of 64KB pages to allocate for Wasm execution. /// Defaults to `DEFAULT_HEAP_PAGES` if `None` is provided. - pub fn new( - fallback_method: WasmExecutionMethod, - default_heap_pages: Option, - max_runtime_instances: usize, - ) -> Self { + pub fn new(fallback_method: WasmExecutionMethod, max_runtime_instances: usize) -> Self { let extended = D::ExtendHostFunctions::host_functions(); let mut host_functions = sp_io::SubstrateHostFunctions::host_functions() .into_iter() @@ -308,13 +316,8 @@ impl NativeExecutor { // Add the custom host functions provided by the user. host_functions.extend(extended); - let wasm_executor = WasmExecutor::new( - fallback_method, - default_heap_pages, - host_functions, - max_runtime_instances, - None, - ); + let wasm_executor = + WasmExecutor::new(fallback_method, host_functions, max_runtime_instances, None); NativeExecutor { _dummy: Default::default(), @@ -481,6 +484,12 @@ impl CodeExecutor for NativeExecutor { use_native: bool, native_call: Option, ) -> (Result>, bool) { + trace!( + target: "executor", + "call({}) [use_native = {}]", + method, + use_native, + ); let mut used_native = false; let result = self.wasm.with_instance( runtime_code, @@ -656,7 +665,7 @@ mod tests { #[test] fn native_executor_registers_custom_interface() { - let executor = NativeExecutor::::new(WasmExecutionMethod::Interpreted, None, 8); + let executor = NativeExecutor::::new(WasmExecutionMethod::Interpreted, 8); my_interface::HostFunctions::host_functions().iter().for_each(|function| { assert_eq!(executor.wasm.host_functions.iter().filter(|f| f == &function).count(), 2); }); diff --git a/client/executor/src/wasm_runtime.rs b/client/executor/src/wasm_runtime.rs index c55af60b70a9f..062fdf180ad22 100644 --- a/client/executor/src/wasm_runtime.rs +++ b/client/executor/src/wasm_runtime.rs @@ -116,9 +116,10 @@ impl VersionedRuntime { if new_inst { log::debug!( target: "wasm-runtime", - "Allocated WASM instance {}/{}", + "Allocated WASM instance {}/{} [pages = {}]", index + 1, self.instances.len(), + self.heap_pages, ); } } @@ -126,7 +127,11 @@ impl VersionedRuntime { result }, None => { - log::warn!(target: "wasm-runtime", "Ran out of free WASM instances"); + log::warn!( + target: "wasm-runtime", + "Ran out of free WASM instances with {} pages", + self.heap_pages, + ); // Allocate a new instance let instance = self.module.new_instance()?; @@ -138,6 +143,7 @@ impl VersionedRuntime { } const MAX_RUNTIMES: usize = 2; +const MAX_OFFCHAIN_RUNTIME_INSTANCES: usize = 2; /// Cache for the runtimes. /// @@ -148,15 +154,20 @@ const MAX_RUNTIMES: usize = 2; /// values of mutable globals. Follow-up requests to fetch a runtime return this one instance with /// the memory reset to the initial memory. So, one runtime instance is reused for every fetch /// request. -/// -/// The size of cache is equal to `MAX_RUNTIMES`. pub struct RuntimeCache { /// A cache of runtimes along with metadata. /// /// Runtimes sorted by recent usage. The most recently used is at the front. - runtimes: Mutex<[Option>; MAX_RUNTIMES]>, + /// + /// These runtime instances are only used in consensus code context and their count is limited + /// by `max_consensus_runtime_instances`. + consensus_runtimes: Mutex<[Option>; MAX_RUNTIMES]>, /// The size of the instances cache for each runtime. - max_runtime_instances: usize, + max_consensus_runtime_instances: usize, + /// Same as `consensus_runtimes`, but used in offchain code context. Their size is always bound + /// by 1 (can easily be configurable, but we don't need this right now). + offchain_runtimes: Mutex<[Option>; MAX_RUNTIMES]>, + /// Optional path used for caching artifacts. cache_path: Option, } @@ -168,8 +179,16 @@ impl RuntimeCache { /// /// `cache_path` allows to specify an optional directory where the executor can store files /// for caching. - pub fn new(max_runtime_instances: usize, cache_path: Option) -> RuntimeCache { - RuntimeCache { runtimes: Default::default(), max_runtime_instances, cache_path } + pub fn new( + max_consensus_runtime_instances: usize, + cache_path: Option, + ) -> RuntimeCache { + RuntimeCache { + consensus_runtimes: Default::default(), + offchain_runtimes: Default::default(), + max_consensus_runtime_instances, + cache_path, + } } /// Prepares a WASM module instance and executes given function for it. @@ -221,7 +240,17 @@ impl RuntimeCache { let code_hash = &runtime_code.hash; let heap_pages = runtime_code.heap_pages.unwrap_or(default_heap_pages); - let mut runtimes = self.runtimes.lock(); // this must be released prior to calling f + // this must be released prior to calling f. + let mut runtimes = match runtime_code.context { + sp_core::traits::CodeContext::Consensus => self.consensus_runtimes.lock(), + sp_core::traits::CodeContext::Offchain => self.offchain_runtimes.lock(), + }; + + let max_runtime_instances = match runtime_code.context { + sp_core::traits::CodeContext::Consensus => self.max_consensus_runtime_instances, + sp_core::traits::CodeContext::Offchain => MAX_OFFCHAIN_RUNTIME_INSTANCES, + }; + let pos = runtimes.iter().position(|r| { r.as_ref().map_or(false, |r| { r.wasm_method == wasm_method && @@ -248,7 +277,7 @@ impl RuntimeCache { heap_pages, host_functions.into(), allow_missing_func_imports, - self.max_runtime_instances, + max_runtime_instances, self.cache_path.as_deref(), ); @@ -257,8 +286,9 @@ impl RuntimeCache { #[cfg(not(target_os = "unknown"))] log::debug!( target: "wasm-runtime", - "Prepared new runtime version {:?} in {} ms.", + "Prepared new runtime version {:?} for {:?} in {} ms.", result.version, + runtime_code.context, time.elapsed().as_millis(), ); }, diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs index 2a10dfc0d50d8..19b29a89b1e66 100644 --- a/client/finality-grandpa/src/lib.rs +++ b/client/finality-grandpa/src/lib.rs @@ -505,7 +505,7 @@ where &BlockId::Number(Zero::zero()), "GrandpaApi_grandpa_authorities", &[], - ExecutionStrategy::NativeElseWasm, + ExecutionStrategy::NativeElseWasm.in_consensus(), None, ) .and_then(|call_result| { diff --git a/client/light/src/call_executor.rs b/client/light/src/call_executor.rs index 144e0cbf96dcc..d88423a8f18dc 100644 --- a/client/light/src/call_executor.rs +++ b/client/light/src/call_executor.rs @@ -33,8 +33,8 @@ use sp_runtime::{ traits::{Block as BlockT, Header as HeaderT}, }; use sp_state_machine::{ - create_proof_check_backend, execution_proof_check_on_trie_backend, ExecutionManager, - ExecutionStrategy, OverlayedChanges, StorageProof, + create_proof_check_backend, execution_proof_check_on_trie_backend, ExecutionConfig, + ExecutionManager, ExecutionStrategy, OverlayedChanges, StorageProof, }; use sp_api::{ProofRecorder, StorageTransactionCache}; @@ -82,7 +82,7 @@ where id: &BlockId, method: &str, call_data: &[u8], - strategy: ExecutionStrategy, + strategy: ExecutionConfig, extensions: Option, ) -> ClientResult> { if self.backend.is_local_state_available(id) { @@ -132,7 +132,7 @@ where call_data, changes, None, - ExecutionManager::NativeWhenPossible, + ExecutionStrategy::NativeWhenPossible.in_consensus().get_manager(), native_call, recorder, extensions, @@ -193,7 +193,7 @@ where // TODO: Remove when solved: https://github.com/paritytech/substrate/issues/5047 let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&trie_backend); let runtime_code = backend_runtime_code - .runtime_code() + .runtime_code(sp_core::traits::CodeContext::Consensus) .map_err(|_e| ClientError::RuntimeCodeMissing)?; // execute method diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index 313e89bdf80b4..7555463a002e1 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -292,7 +292,7 @@ where &BlockId::Hash(block), &method, &*call_data, - self.client.execution_extensions().strategies().other, + self.client.execution_extensions().configs().other, None, ) .map(Into::into) diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 83c8e1d9d1cbe..99fa110ff1571 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -291,11 +291,8 @@ where TaskManager::new(config.task_executor.clone(), registry)? }; - let executor = NativeExecutor::::new( - config.wasm_method, - config.default_heap_pages, - config.max_runtime_instances, - ); + let executor = + NativeExecutor::::new(config.wasm_method, config.max_runtime_instances); let chain_spec = &config.chain_spec; let fork_blocks = get_extension::>(chain_spec.extensions()) @@ -319,7 +316,7 @@ where let backend = new_db_backend(db_config)?; let extensions = sc_client_api::execution_extensions::ExecutionExtensions::new( - config.execution_strategies.clone(), + config.execution_configs.clone(), Some(keystore_container.sync_keystore()), sc_offchain::OffchainDb::factory_from_backend(&*backend), ); @@ -382,11 +379,8 @@ where TaskManager::new(config.task_executor.clone(), registry)? }; - let executor = NativeExecutor::::new( - config.wasm_method, - config.default_heap_pages, - config.max_runtime_instances, - ); + let executor = + NativeExecutor::::new(config.wasm_method, config.max_runtime_instances); let db_storage = { let db_settings = sc_client_db::DatabaseSettings { diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 2fae972d3472d..cc44d16220045 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -31,8 +31,8 @@ use sp_runtime::{ traits::{Block as BlockT, NumberFor}, }; use sp_state_machine::{ - self, backend::Backend as _, ExecutionManager, ExecutionStrategy, Ext, OverlayedChanges, - StateMachine, StorageProof, + self, backend::Backend as _, ExecutionManager, Ext, OverlayedChanges, StateMachine, + StorageProof, }; use std::{cell::RefCell, panic::UnwindSafe, result, sync::Arc}; @@ -81,9 +81,9 @@ where }) } - /// Check if local runtime code overrides are enabled and one is available - /// for the given `BlockId`. If yes, return it; otherwise return the same - /// `RuntimeCode` instance that was passed. + /// Check if local runtime code overrides are enabled and one is available for the given + /// `BlockId`. If yes, return it; otherwise return the same `RuntimeCode` instance that was + /// passed. fn check_override<'a>( &'a self, onchain_code: RuntimeCode<'a>, @@ -97,12 +97,15 @@ where let code = if let Some(d) = self .wasm_override .as_ref() - .map(|o| o.get(&spec, onchain_code.heap_pages)) + .map(|o| o.get(&spec, onchain_code.heap_pages, onchain_code.context)) .flatten() { log::debug!(target: "wasm_overrides", "using WASM override for block {}", id); d - } else if let Some(s) = self.wasm_substitutes.get(spec, onchain_code.heap_pages, id) { + } else if let Some(s) = + self.wasm_substitutes + .get(spec, onchain_code.heap_pages, onchain_code.context, id) + { log::debug!(target: "wasm_substitutes", "Using WASM substitute for block {:?}", id); s } else { @@ -149,7 +152,7 @@ where at: &BlockId, method: &str, call_data: &[u8], - strategy: ExecutionStrategy, + config: sp_state_machine::ExecutionConfig, extensions: Option, ) -> sp_blockchain::Result> { let mut changes = OverlayedChanges::default(); @@ -157,8 +160,9 @@ where backend::changes_tries_state_at_block(at, self.backend.changes_trie_storage())?; let state = self.backend.state_at(*at)?; let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); - let runtime_code = - state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; + let runtime_code = state_runtime_code + .runtime_code(config.context) + .map_err(sp_blockchain::Error::RuntimeCode)?; let runtime_code = self.check_override(runtime_code, at)?; @@ -179,7 +183,7 @@ where ) .set_parent_hash(at_hash) .execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( - strategy.get_manager(), + config.get_manager(), None, )?; @@ -213,7 +217,6 @@ where let mut storage_transaction_cache = storage_transaction_cache.map(|c| c.borrow_mut()); let mut state = self.backend.state_at(*at)?; - let changes = &mut *changes.borrow_mut(); let at_hash = self.backend.blockchain().block_hash_from_id(at)?.ok_or_else(|| { @@ -231,8 +234,9 @@ where sp_state_machine::backend::BackendRuntimeCode::new(trie_state); // It is important to extract the runtime code here before we create the proof // recorder. - let runtime_code = - state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; + let runtime_code = state_runtime_code + .runtime_code(execution_manager.context) + .map_err(sp_blockchain::Error::RuntimeCode)?; let runtime_code = self.check_override(runtime_code, at)?; let backend = sp_state_machine::ProvingBackend::new_with_recorder( @@ -261,8 +265,9 @@ where }, None => { let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); - let runtime_code = - state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; + let runtime_code = state_runtime_code + .runtime_code(execution_manager.context) + .map_err(sp_blockchain::Error::RuntimeCode)?; let runtime_code = self.check_override(runtime_code, at)?; let mut state_machine = StateMachine::new( @@ -297,8 +302,9 @@ where let mut cache = StorageTransactionCache::::default(); let mut ext = Ext::new(&mut overlay, &mut cache, &state, changes_trie_state, None); let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); - let runtime_code = - state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; + let runtime_code = state_runtime_code + .runtime_code(sp_core::traits::CodeContext::Consensus) + .map_err(sp_blockchain::Error::RuntimeCode)?; self.executor .runtime_version(&mut ext, &runtime_code) .map_err(|e| sp_blockchain::Error::VersionInvalid(format!("{:?}", e)).into()) @@ -318,8 +324,9 @@ where })?; let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(trie_backend); - let runtime_code = - state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; + let runtime_code = state_runtime_code + .runtime_code(sp_core::traits::CodeContext::Consensus) + .map_err(sp_blockchain::Error::RuntimeCode)?; let runtime_code = self.check_override(runtime_code, at)?; sp_state_machine::prove_execution_on_trie_backend::<_, _, NumberFor, _, _>( @@ -367,14 +374,14 @@ mod tests { #[test] fn should_get_override_if_exists() { - let executor = - NativeExecutor::::new(WasmExecutionMethod::Interpreted, Some(128), 1); + let executor = NativeExecutor::::new(WasmExecutionMethod::Interpreted, 1); let overrides = crate::client::wasm_override::dummy_overrides(&executor); let onchain_code = WrappedRuntimeCode(substrate_test_runtime::wasm_binary_unwrap().into()); let onchain_code = RuntimeCode { code_fetcher: &onchain_code, heap_pages: Some(128), + context: sp_core::traits::CodeContext::Consensus, hash: vec![0, 0, 0, 0], }; diff --git a/client/service/src/client/wasm_override.rs b/client/service/src/client/wasm_override.rs index 7abd04f2be236..982f2b6fba382 100644 --- a/client/service/src/client/wasm_override.rs +++ b/client/service/src/client/wasm_override.rs @@ -37,7 +37,7 @@ //! needed must be provided in the given directory. use sc_executor::RuntimeInfo; use sp_blockchain::Result; -use sp_core::traits::{FetchRuntimeCode, RuntimeCode}; +use sp_core::traits::{CodeContext, FetchRuntimeCode, RuntimeCode}; use sp_state_machine::BasicExternalities; use sp_version::RuntimeVersion; use std::{ @@ -60,8 +60,8 @@ impl WasmBlob { Self { code, hash } } - fn runtime_code(&self, heap_pages: Option) -> RuntimeCode { - RuntimeCode { code_fetcher: self, hash: self.hash.clone(), heap_pages } + fn runtime_code(&self, heap_pages: Option, context: CodeContext) -> RuntimeCode { + RuntimeCode { code_fetcher: self, hash: self.hash.clone(), context, heap_pages } } } @@ -125,8 +125,13 @@ where /// Gets an override by it's runtime spec version. /// /// Returns `None` if an override for a spec version does not exist. - pub fn get<'a, 'b: 'a>(&'b self, spec: &u32, pages: Option) -> Option> { - self.overrides.get(spec).map(|w| w.runtime_code(pages)) + pub fn get<'a, 'b: 'a>( + &'b self, + spec: &u32, + pages: Option, + context: CodeContext, + ) -> Option> { + self.overrides.get(spec).map(|w| w.runtime_code(pages, context)) } /// Scrapes a folder for WASM runtimes. @@ -148,7 +153,8 @@ where match path.extension().map(|e| e.to_str()).flatten() { Some("wasm") => { let wasm = WasmBlob::new(fs::read(&path).map_err(handle_err)?); - let version = Self::runtime_version(executor, &wasm, Some(128))?; + let version = + Self::runtime_version(executor, &wasm, Some(128), CodeContext::Consensus)?; log::info!( target: "wasm_overrides", "Found wasm override in file: `{:?}`, version: {}", @@ -180,10 +186,11 @@ where executor: &E, code: &WasmBlob, heap_pages: Option, + context: CodeContext, ) -> Result { let mut ext = BasicExternalities::default(); executor - .runtime_version(&mut ext, &code.runtime_code(heap_pages)) + .runtime_version(&mut ext, &code.runtime_code(heap_pages, context)) .map_err(|e| WasmOverrideError::VersionInvalid(format!("{:?}", e)).into()) } } @@ -214,7 +221,6 @@ mod tests { { let exec = NativeExecutor::::new( WasmExecutionMethod::Interpreted, - Some(128), 1, ); let bytes = substrate_test_runtime::wasm_binary_unwrap(); @@ -226,11 +232,11 @@ mod tests { #[test] fn should_get_runtime_version() { let wasm = WasmBlob::new(substrate_test_runtime::wasm_binary_unwrap().to_vec()); - let executor = - NativeExecutor::::new(WasmExecutionMethod::Interpreted, Some(128), 1); + let executor = NativeExecutor::::new(WasmExecutionMethod::Interpreted, 1); - let version = WasmOverride::runtime_version(&executor, &wasm, Some(128)) - .expect("should get the `RuntimeVersion` of the test-runtime wasm blob"); + let version = + WasmOverride::runtime_version(&executor, &wasm, Some(128), CodeContext::Consensus) + .expect("should get the `RuntimeVersion` of the test-runtime wasm blob"); assert_eq!(version.spec_version, 2); } diff --git a/client/service/src/client/wasm_substitutes.rs b/client/service/src/client/wasm_substitutes.rs index ac48059fc2f37..1a9b6ef1740bb 100644 --- a/client/service/src/client/wasm_substitutes.rs +++ b/client/service/src/client/wasm_substitutes.rs @@ -57,8 +57,12 @@ impl WasmSubstitute { Ok(Self { code, hash, block_hash, block_number }) } - fn runtime_code(&self, heap_pages: Option) -> RuntimeCode { - RuntimeCode { code_fetcher: self, hash: self.hash.clone(), heap_pages } + fn runtime_code( + &self, + heap_pages: Option, + context: sp_core::traits::CodeContext, + ) -> RuntimeCode { + RuntimeCode { code_fetcher: self, hash: self.hash.clone(), heap_pages, context } } /// Returns `true` when the substitute matches for the given `block_id`. @@ -168,10 +172,11 @@ where &self, spec: u32, pages: Option, + context: sp_core::traits::CodeContext, block_id: &BlockId, ) -> Option> { let s = self.substitutes.get(&spec)?; - s.matches(block_id, &*self.backend).then(|| s.runtime_code(pages)) + s.matches(block_id, &*self.backend).then(|| s.runtime_code(pages, context)) } fn runtime_version( @@ -180,7 +185,10 @@ where ) -> Result { let mut ext = BasicExternalities::default(); executor - .runtime_version(&mut ext, &code.runtime_code(None)) + .runtime_version( + &mut ext, + &code.runtime_code(None, sp_core::traits::CodeContext::Consensus), + ) .map_err(|e| WasmSubstituteError::VersionInvalid(format!("{:?}", e)).into()) } } diff --git a/client/service/src/config.rs b/client/service/src/config.rs index c915978f5384e..b301404cdfbf5 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -18,7 +18,7 @@ //! Service configuration. -pub use sc_client_api::execution_extensions::{ExecutionStrategies, ExecutionStrategy}; +pub use sc_client_api::execution_extensions::ExecutionStrategy; pub use sc_client_db::{ Database, DatabaseSettingsSrc as DatabaseConfig, KeepBlocks, PruningMode, TransactionStorageMode, @@ -35,6 +35,7 @@ pub use sc_network::{ use prometheus_endpoint::Registry; use sc_chain_spec::ChainSpec; +pub use sc_client_api::execution_extensions::ExecutionConfigs; pub use sc_telemetry::TelemetryEndpoints; pub use sc_transaction_pool::Options as TransactionPoolOptions; use sp_core::crypto::SecretString; @@ -88,8 +89,8 @@ pub struct Configuration { /// over on-chain runtimes when the spec version matches. Set to `None` to /// disable overrides (default). pub wasm_runtime_overrides: Option, - /// Execution strategies. - pub execution_strategies: ExecutionStrategies, + /// Execution configurations. + pub execution_configs: ExecutionConfigs, /// RPC over HTTP binding address. `None` if disabled. pub rpc_http: Option, /// RPC over Websockets binding address. `None` if disabled. @@ -113,8 +114,6 @@ pub struct Configuration { /// External WASM transport for the telemetry. If `Some`, when connection to a telemetry /// endpoint, this transport will be tried in priority before all others. pub telemetry_external_transport: Option, - /// The default number of 64KB pages to allocate for Wasm execution - pub default_heap_pages: Option, /// Should offchain workers be executed. pub offchain_worker: OffchainWorkerConfig, /// Enable authoring even when offline. @@ -123,7 +122,8 @@ pub struct Configuration { pub disable_grandpa: bool, /// Development key seed. /// - /// When running in development mode, the seed will be used to generate authority keys by the keystore. + /// When running in development mode, the seed will be used to generate authority keys by the + /// keystore. /// /// Should only be set when `node` is running development mode. pub dev_key_seed: Option, diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 90f87670c0cec..ec28892ddf532 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -206,7 +206,7 @@ impl CallExecutor for DummyCallExecutor { _id: &BlockId, _method: &str, _call_data: &[u8], - _strategy: ExecutionStrategy, + _config: sp_state_machine::ExecutionConfig, _extensions: Option, ) -> Result, ClientError> { Ok(vec![42]) @@ -263,7 +263,7 @@ impl CallExecutor for DummyCallExecutor { } fn local_executor() -> NativeExecutor { - NativeExecutor::new(WasmExecutionMethod::Interpreted, None, 8) + NativeExecutor::new(WasmExecutionMethod::Interpreted, 8) } #[test] @@ -430,7 +430,13 @@ fn code_is_executed_at_genesis_only() { let genesis_executor = GenesisCallExecutor::new(backend, DummyCallExecutor); assert_eq!( genesis_executor - .call(&BlockId::Number(0), "test_method", &[], ExecutionStrategy::NativeElseWasm, None,) + .call( + &BlockId::Number(0), + "test_method", + &[], + ExecutionStrategy::NativeElseWasm.in_consensus(), + None, + ) .unwrap(), vec![42], ); @@ -439,7 +445,7 @@ fn code_is_executed_at_genesis_only() { &BlockId::Number(1), "test_method", &[], - ExecutionStrategy::NativeElseWasm, + ExecutionStrategy::NativeElseWasm.in_consensus(), None, ); diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 6ac149677bc11..672cf1c43ee89 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -70,7 +70,7 @@ native_executor_instance!( ); fn executor() -> sc_executor::NativeExecutor { - sc_executor::NativeExecutor::new(sc_executor::WasmExecutionMethod::Interpreted, None, 8) + sc_executor::NativeExecutor::new(sc_executor::WasmExecutionMethod::Interpreted, 8) } pub fn prepare_client_with_key_changes() -> ( @@ -175,7 +175,9 @@ fn construct_block( let hash = header.hash(); let mut overlay = OverlayedChanges::default(); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(backend); - let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); + let runtime_code = backend_runtime_code + .runtime_code(sp_core::traits::CodeContext::Consensus) + .expect("Code is part of the backend"); let task_executor = Box::new(TaskExecutor::new()); StateMachine::new( @@ -189,7 +191,7 @@ fn construct_block( &runtime_code, task_executor.clone() as Box<_>, ) - .execute(ExecutionStrategy::NativeElseWasm) + .execute(ExecutionStrategy::NativeElseWasm.in_consensus()) .unwrap(); for tx in transactions.iter() { @@ -204,7 +206,7 @@ fn construct_block( &runtime_code, task_executor.clone() as Box<_>, ) - .execute(ExecutionStrategy::NativeElseWasm) + .execute(ExecutionStrategy::NativeElseWasm.in_consensus()) .unwrap(); } @@ -219,7 +221,7 @@ fn construct_block( &runtime_code, task_executor.clone() as Box<_>, ) - .execute(ExecutionStrategy::NativeElseWasm) + .execute(ExecutionStrategy::NativeElseWasm.in_consensus()) .unwrap(); header = Header::decode(&mut &ret_data[..]).unwrap(); @@ -257,7 +259,9 @@ fn construct_genesis_should_work_with_native() { let backend = InMemoryBackend::from(storage); let (b1data, _b1hash) = block1(genesis_hash, &backend); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); - let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); + let runtime_code = backend_runtime_code + .runtime_code(sp_core::traits::CodeContext::Consensus) + .expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); @@ -272,7 +276,7 @@ fn construct_genesis_should_work_with_native() { &runtime_code, TaskExecutor::new(), ) - .execute(ExecutionStrategy::NativeElseWasm) + .execute(ExecutionStrategy::NativeElseWasm.in_consensus()) .unwrap(); } @@ -292,7 +296,9 @@ fn construct_genesis_should_work_with_wasm() { let backend = InMemoryBackend::from(storage); let (b1data, _b1hash) = block1(genesis_hash, &backend); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); - let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); + let runtime_code = backend_runtime_code + .runtime_code(sp_core::traits::CodeContext::Consensus) + .expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); @@ -307,7 +313,7 @@ fn construct_genesis_should_work_with_wasm() { &runtime_code, TaskExecutor::new(), ) - .execute(ExecutionStrategy::AlwaysWasm) + .execute(ExecutionStrategy::AlwaysWasm.in_consensus()) .unwrap(); } @@ -327,7 +333,9 @@ fn construct_genesis_with_bad_transaction_should_panic() { let backend = InMemoryBackend::from(storage); let (b1data, _b1hash) = block1(genesis_hash, &backend); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); - let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); + let runtime_code = backend_runtime_code + .runtime_code(sp_core::traits::CodeContext::Consensus) + .expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); @@ -342,7 +350,7 @@ fn construct_genesis_with_bad_transaction_should_panic() { &runtime_code, TaskExecutor::new(), ) - .execute(ExecutionStrategy::NativeElseWasm); + .execute(ExecutionStrategy::NativeElseWasm.in_consensus()); assert!(r.is_err()); } diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index 9433ed0bde06f..030e18a34ace7 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -245,7 +245,7 @@ fn node_config< chain_spec: Box::new((*spec).clone()), wasm_method: sc_service::config::WasmExecutionMethod::Interpreted, wasm_runtime_overrides: Default::default(), - execution_strategies: Default::default(), + execution_configs: Default::default(), rpc_http: None, rpc_ipc: None, rpc_ws: None, @@ -257,7 +257,6 @@ fn node_config< prometheus_config: None, telemetry_endpoints: None, telemetry_external_transport: None, - default_heap_pages: None, offchain_worker: Default::default(), force_authoring: false, disable_grandpa: false, diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 68681ea5aca60..53291fa1727c6 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -27,8 +27,8 @@ //! The System pallet defines the core data types used in a Substrate runtime. //! It also provides several utility functions (see [`Pallet`]) for other FRAME pallets. //! -//! In addition, it manages the storage items for extrinsics data, indexes, event records, and digest items, -//! among other things that support the execution of the current block. +//! In addition, it manages the storage items for extrinsics data, indexes, event records, and +//! digest items, among other things that support the execution of the current block. //! //! It also handles low-level tasks like depositing logs, basic set up and take down of //! temporary storage entries, and access to previous block hashes. @@ -54,10 +54,10 @@ //! - [`CheckEra`]: Checks the era of the transaction. Contains a single payload of type `Era`. //! - [`CheckGenesis`]: Checks the provided genesis hash of the transaction. Must be a part of the //! signed payload of the transaction. -//! - [`CheckSpecVersion`]: Checks that the runtime version is the same as the one used to sign the -//! transaction. -//! - [`CheckTxVersion`]: Checks that the transaction version is the same as the one used to sign the -//! transaction. +//! - [`CheckSpecVersion`]: Checks that the runtime version is the same as the one used to sign +//! the transaction. +//! - [`CheckTxVersion`]: Checks that the transaction version is the same as the one used to sign +//! the transaction. //! //! Lookup the runtime aggregator file (e.g. `node/runtime`) to see the full list of signed //! extensions included in a chain. @@ -343,14 +343,7 @@ pub mod pallet { Ok(().into()) } - /// Set the number of pages in the WebAssembly environment's heap. - /// - /// # - /// - `O(1)` - /// - 1 storage write. - /// - Base Weight: 1.405 µs - /// - 1 write to HEAP_PAGES - /// # + /// Set the number of pages in the consensus WebAssembly environment's heap. #[pallet::weight((T::SystemWeightInfo::set_heap_pages(), DispatchClass::Operational))] pub fn set_heap_pages(origin: OriginFor, pages: u64) -> DispatchResultWithPostInfo { ensure_root(origin)?; @@ -497,11 +490,6 @@ pub mod pallet { } /// Make some on-chain remark and emit event. - /// - /// # - /// - `O(b)` where b is the length of the remark. - /// - 1 event. - /// # #[pallet::weight(T::SystemWeightInfo::remark_with_event(remark.len() as u32))] pub fn remark_with_event( origin: OriginFor, @@ -512,6 +500,14 @@ pub mod pallet { Self::deposit_event(Event::Remarked(who, hash)); Ok(().into()) } + + /// Set the number of pages in the offchain WebAssembly environment's heap. + #[pallet::weight((T::SystemWeightInfo::set_heap_pages(), DispatchClass::Operational))] + pub fn set_heap_pages_offchain(origin: OriginFor, pages: u64) -> DispatchResult { + ensure_root(origin)?; + storage::unhashed::put_raw(well_known_keys::OFFCHAIN_HEAP_PAGES, &pages.encode()); + Ok(().into()) + } } /// Event for the System pallet. diff --git a/primitives/api/test/tests/runtime_calls.rs b/primitives/api/test/tests/runtime_calls.rs index b0b14ec1e944e..9bb8a88b73703 100644 --- a/primitives/api/test/tests/runtime_calls.rs +++ b/primitives/api/test/tests/runtime_calls.rs @@ -179,6 +179,7 @@ fn record_proof_works() { code_fetcher: &sp_core::traits::WrappedRuntimeCode( client.code_at(&block_id).unwrap().into(), ), + context: sp_core::traits::CodeContext::Consensus, hash: vec![1], heap_pages: None, }; @@ -206,7 +207,7 @@ fn record_proof_works() { // Use the proof backend to execute `execute_block`. let mut overlay = Default::default(); - let executor = NativeExecutor::::new(WasmExecutionMethod::Interpreted, None, 8); + let executor = NativeExecutor::::new(WasmExecutionMethod::Interpreted, 8); execution_proof_check_on_trie_backend::<_, u64, _, _>( &backend, &mut overlay, diff --git a/primitives/core/src/traits.rs b/primitives/core/src/traits.rs index dfa61f606cb9e..c14547d3ab588 100644 --- a/primitives/core/src/traits.rs +++ b/primitives/core/src/traits.rs @@ -72,6 +72,17 @@ impl FetchRuntimeCode for NoneFetchRuntimeCode { } } +/// The context of the runtime code. +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum CodeContext { + /// Consensus critical task. Resources should be sufficient, but not not more, and trust + /// levels should be high. The need for determinism is absolute. + Consensus, + /// Offchain operations that do not affect the chain state directly. Can be used for + /// offchain worker threads, and other utility tools. + Offchain, +} + /// The Wasm code of a Substrate runtime. #[derive(Clone)] pub struct RuntimeCode<'a> { @@ -79,8 +90,10 @@ pub struct RuntimeCode<'a> { pub code_fetcher: &'a dyn FetchRuntimeCode, /// The optional heap pages this `code` should be executed with. /// - /// If `None` are given, the default value of the executor will be used. + /// If `None` is given, the default value of the executor will be used. pub heap_pages: Option, + /// The context at which this code is intended to be executed. + pub context: CodeContext, /// The SCALE encoded hash of `code`. /// /// The hashing algorithm isn't that important, as long as all runtime @@ -99,7 +112,12 @@ impl<'a> RuntimeCode<'a> { /// /// This is only useful for tests that don't want to execute any code. pub fn empty() -> Self { - Self { code_fetcher: &NoneFetchRuntimeCode, hash: Vec::new(), heap_pages: None } + Self { + code_fetcher: &NoneFetchRuntimeCode, + hash: Vec::new(), + heap_pages: None, + context: CodeContext::Consensus, + } } } diff --git a/primitives/runtime-interface/test/src/lib.rs b/primitives/runtime-interface/test/src/lib.rs index 82c50fffeb8d7..21c016d723162 100644 --- a/primitives/runtime-interface/test/src/lib.rs +++ b/primitives/runtime-interface/test/src/lib.rs @@ -44,7 +44,6 @@ fn call_wasm_method_with_result( let executor = sc_executor::WasmExecutor::new( sc_executor::WasmExecutionMethod::Interpreted, - Some(8), host_functions, 8, None, @@ -56,6 +55,7 @@ fn call_wasm_method_with_result( false, method, &[], + 8, ) .map_err(|e| format!("Failed to execute `{}`: {}", method, e))?; Ok(ext) diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index de4ff33b51fe8..3df68eb08ce7a 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -345,7 +345,10 @@ where } /// Return the [`RuntimeCode`] build from the wrapped `backend`. - pub fn runtime_code(&self) -> Result { + pub fn runtime_code( + &self, + context: sp_core::traits::CodeContext, + ) -> Result { let hash = self .backend .storage_hash(well_known_keys::CODE) @@ -353,13 +356,18 @@ where .flatten() .ok_or("`:code` hash not found")? .encode(); + + let heap_pages_key = match context { + sp_core::traits::CodeContext::Consensus => well_known_keys::HEAP_PAGES, + sp_core::traits::CodeContext::Offchain => well_known_keys::OFFCHAIN_HEAP_PAGES, + }; let heap_pages = self .backend - .storage(well_known_keys::HEAP_PAGES) + .storage(heap_pages_key) .ok() .flatten() .and_then(|d| Decode::decode(&mut &d[..]).ok()); - Ok(RuntimeCode { code_fetcher: self, hash, heap_pages }) + Ok(RuntimeCode { code_fetcher: self, hash, context, heap_pages }) } } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 1588a42f41fec..79516087ebff6 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -175,7 +175,7 @@ mod execution { use sp_core::{ hexdisplay::HexDisplay, storage::ChildInfo, - traits::{CodeExecutor, ReadRuntimeVersionExt, RuntimeCode, SpawnNamed}, + traits::{CodeContext, CodeExecutor, ReadRuntimeVersionExt, RuntimeCode, SpawnNamed}, NativeOrEncoded, NeverNativeValue, }; use sp_externalities::Extensions; @@ -198,6 +198,17 @@ mod execution { /// Trie backend with in-memory storage. pub type InMemoryBackend = TrieBackend, H>; + /// Configurations of an execution. + /// + /// Combination of a strategy, and a context. + #[derive(Copy, Clone, Eq, PartialEq, Debug)] + pub struct ExecutionConfig { + /// The strategy of the execution's runtime. + pub strategy: ExecutionStrategy, + /// The context at which the aforementioned strategy is being used. + pub context: CodeContext, + } + /// Strategy for executing a call into the runtime. #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum ExecutionStrategy { @@ -206,86 +217,157 @@ mod execution { NativeWhenPossible, /// Use the given wasm module. AlwaysWasm, - /// Run with both the wasm and the native variant (if compatible). Report any discrepancy as an error. + /// Run with both the wasm and the native variant (if compatible). Report any discrepancy + /// as an error. Both, /// First native, then if that fails or is not possible, wasm. NativeElseWasm, } + impl ExecutionStrategy { + /// Consumes self and returned the corresponding [`ExecutionConfig`], when executed in a + /// consensus context. + /// + /// This is merely a syntactic sugar around [`ExecutionConfig::new_consensus`]. + pub fn in_consensus(self) -> ExecutionConfig { + ExecutionConfig::new_consensus(self) + } + + /// Consumes self and returned the corresponding [`ExecutionConfig`], when executed in a + /// offchain context. + /// + /// This is merely a syntactic sugar around [`ExecutionConfig::new_offchain`]. + pub fn in_offchain(self) -> ExecutionConfig { + ExecutionConfig::new_offchain(self) + } + } + + impl ExecutionConfig { + /// Build a new [`ExecutionConfig`] with the given strategy and + /// [`CodeContext::Consensus`]. + pub fn new_consensus(strategy: ExecutionStrategy) -> Self { + Self { strategy, context: CodeContext::Consensus } + } + + /// Build a new [`ExecutionConfig`] with the given strategy and + /// [`CodeContext::Offchain`]. + pub fn new_offchain(strategy: ExecutionStrategy) -> Self { + Self { strategy, context: CodeContext::Offchain } + } + + /// Gets the corresponding manager for the execution config. + pub fn get_manager( + self, + ) -> ExecutionManager> { + let failure_handler: DefaultHandler = |wasm_result, native_result| { + warn!( + "Consensus error between wasm {:?} and native {:?}. Using wasm.", + wasm_result, native_result, + ); + warn!(" Native result {:?}", native_result); + warn!(" Wasm result {:?}", wasm_result); + wasm_result + }; + + let strategy = match self.strategy { + ExecutionStrategy::AlwaysWasm => + ExecutionStrategyWithHandler::AlwaysWasm(BackendTrustLevel::Trusted), + ExecutionStrategy::NativeWhenPossible => + ExecutionStrategyWithHandler::NativeWhenPossible, + ExecutionStrategy::NativeElseWasm => ExecutionStrategyWithHandler::NativeElseWasm, + ExecutionStrategy::Both => ExecutionStrategyWithHandler::Both(failure_handler), + }; + let context = self.context; + + ExecutionManager { strategy, context } + } + } + /// Storage backend trust level. #[derive(Debug, Clone)] pub enum BackendTrustLevel { /// Panics from trusted backends are considered justified, and never caught. Trusted, - /// Panics from untrusted backend are caught and interpreted as runtime error. - /// Untrusted backend may be missing some parts of the trie, so panics are not considered - /// fatal. + /// Panics from untrusted backend are caught and interpreted as runtime error. Untrusted + /// backend may be missing some parts of the trie, so panics are not considered fatal. Untrusted, } - /// Like `ExecutionStrategy` only it also stores a handler in case of consensus failure. + /// Like `ExecutionStrategy`, only it also stores a handler in case of consensus failure. #[derive(Clone)] - pub enum ExecutionManager { + pub enum ExecutionStrategyWithHandler { /// Execute with the native equivalent if it is compatible with the given wasm module; /// otherwise fall back to the wasm. NativeWhenPossible, - /// Use the given wasm module. The backend on which code is executed code could be - /// trusted to provide all storage or not (i.e. the light client cannot be trusted to provide - /// for all storage queries since the storage entries it has come from an external node). + /// Use the given wasm module. The backend on which code is executed code could be trusted + /// to provide all storage or not (i.e. the light client cannot be trusted to provide for + /// all storage queries since the storage entries it has come from an external node). AlwaysWasm(BackendTrustLevel), - /// Run with both the wasm and the native variant (if compatible). Call `F` in the case of any discrepancy. + /// Run with both the wasm and the native variant (if compatible). Call `F` in the case of + /// any discrepancy. Both(F), /// First native, then if that fails or is not possible, wasm. NativeElseWasm, } - impl<'a, F> From<&'a ExecutionManager> for ExecutionStrategy { - fn from(s: &'a ExecutionManager) -> Self { - match *s { - ExecutionManager::NativeWhenPossible => ExecutionStrategy::NativeWhenPossible, - ExecutionManager::AlwaysWasm(_) => ExecutionStrategy::AlwaysWasm, - ExecutionManager::NativeElseWasm => ExecutionStrategy::NativeElseWasm, - ExecutionManager::Both(_) => ExecutionStrategy::Both, - } + /// The execution manager. + /// + /// Same as [`ExecutionConfig`], but the inner `strategy` contains failure handler as well. + #[derive(Clone)] + pub struct ExecutionManager { + /// The strategy of the execution's runtime. + pub strategy: ExecutionStrategyWithHandler, + /// The context in which the aforementioned strategy is being used. + pub context: CodeContext, + } + + #[cfg(test)] + impl ExecutionManager { + /// Create a new instance of [`Self`] from the given strategy for testing. + pub(crate) fn new(strategy: ExecutionStrategyWithHandler) -> Self { + Self { strategy, context: CodeContext::Consensus } } } - impl ExecutionStrategy { - /// Gets the corresponding manager for the execution strategy. - pub fn get_manager( - self, - ) -> ExecutionManager> { - match self { - ExecutionStrategy::AlwaysWasm => - ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted), - ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible, - ExecutionStrategy::NativeElseWasm => ExecutionManager::NativeElseWasm, - ExecutionStrategy::Both => ExecutionManager::Both(|wasm_result, native_result| { - warn!( - "Consensus error between wasm {:?} and native {:?}. Using wasm.", - wasm_result, native_result, - ); - warn!(" Native result {:?}", native_result); - warn!(" Wasm result {:?}", wasm_result); - wasm_result - }), - } + impl<'a, F> From<&'a ExecutionManager> for ExecutionConfig { + fn from(s: &'a ExecutionManager) -> Self { + let strategy = match s.strategy { + ExecutionStrategyWithHandler::NativeWhenPossible => + ExecutionStrategy::NativeWhenPossible, + ExecutionStrategyWithHandler::AlwaysWasm(_) => ExecutionStrategy::AlwaysWasm, + ExecutionStrategyWithHandler::NativeElseWasm => ExecutionStrategy::NativeElseWasm, + ExecutionStrategyWithHandler::Both(_) => ExecutionStrategy::Both, + }; + let context = s.context; + + Self { strategy, context } } } - /// Evaluate to ExecutionManager::NativeElseWasm, without having to figure out the type. + /// Helper to create the [`ExecutionStrategyWithHandler::NativeElseWasm`] without naming types. pub fn native_else_wasm() -> ExecutionManager> { - ExecutionManager::NativeElseWasm + ExecutionManager { + strategy: ExecutionStrategyWithHandler::NativeElseWasm, + context: CodeContext::Consensus, + } } - /// Evaluate to ExecutionManager::AlwaysWasm with trusted backend, without having to figure out the type. + /// Helper to create the [`ExecutionStrategyWithHandler::AlwaysWasm(Trusted)`] without naming + /// types. fn always_wasm() -> ExecutionManager> { - ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted) + ExecutionManager { + strategy: ExecutionStrategyWithHandler::AlwaysWasm(BackendTrustLevel::Trusted), + context: CodeContext::Consensus, + } } - /// Evaluate ExecutionManager::AlwaysWasm with untrusted backend, without having to figure out the type. + /// Helper to create the [`ExecutionStrategyWithHandler::AlwaysWasm(Untrusted)`] without naming + /// types. fn always_untrusted_wasm() -> ExecutionManager> { - ExecutionManager::AlwaysWasm(BackendTrustLevel::Untrusted) + ExecutionManager { + strategy: ExecutionStrategyWithHandler::AlwaysWasm(BackendTrustLevel::Untrusted), + context: CodeContext::Consensus, + } } /// The substrate state machine. @@ -389,11 +471,11 @@ mod execution { /// blocks (e.g. a transaction at a time), ensure a different method is used. /// /// Returns the SCALE encoded result of the executed function. - pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result, Box> { - // We are not giving a native call and thus we are sure that the result can never be a native - // value. + pub fn execute(&mut self, config: ExecutionConfig) -> Result, Box> { + // We are not giving a native call and thus we are sure that the result can never be a + // native value. self.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( - strategy.get_manager(), + config.get_manager(), None, ) .map(NativeOrEncoded::into_encoded) @@ -547,12 +629,12 @@ mod execution { self.overlay.set_collect_extrinsics(changes_tries_enabled); let result = { - match manager { - ExecutionManager::Both(on_consensus_failure) => self + match manager.strategy { + ExecutionStrategyWithHandler::Both(on_consensus_failure) => self .execute_call_with_both_strategy(native_call.take(), on_consensus_failure), - ExecutionManager::NativeElseWasm => + ExecutionStrategyWithHandler::NativeElseWasm => self.execute_call_with_native_else_wasm_strategy(native_call.take()), - ExecutionManager::AlwaysWasm(trust_level) => { + ExecutionStrategyWithHandler::AlwaysWasm(trust_level) => { let _abort_guard = match trust_level { BackendTrustLevel::Trusted => None, BackendTrustLevel::Untrusted => @@ -560,7 +642,8 @@ mod execution { }; self.execute_aux(false, native_call).0 }, - ExecutionManager::NativeWhenPossible => self.execute_aux(true, native_call).0, + ExecutionStrategyWithHandler::NativeWhenPossible => + self.execute_aux(true, native_call).0, } }; @@ -1078,7 +1161,12 @@ mod tests { TaskExecutor::new(), ); - assert_eq!(state_machine.execute(ExecutionStrategy::NativeWhenPossible).unwrap(), vec![66]); + assert_eq!( + state_machine + .execute(ExecutionStrategy::NativeWhenPossible.in_consensus()) + .unwrap(), + vec![66], + ); } #[test] @@ -1104,7 +1192,10 @@ mod tests { TaskExecutor::new(), ); - assert_eq!(state_machine.execute(ExecutionStrategy::NativeElseWasm).unwrap(), vec![66]); + assert_eq!( + state_machine.execute(ExecutionStrategy::NativeElseWasm.in_consensus()).unwrap(), + vec![66] + ); } #[test] @@ -1133,10 +1224,10 @@ mod tests { assert!(state_machine .execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( - ExecutionManager::Both(|we, _ne| { + ExecutionManager::new(ExecutionStrategyWithHandler::Both(|we, _ne| { consensus_failed = true; we - }), + })), None, ) .is_err()); @@ -1752,7 +1843,7 @@ mod tests { let run_state_machine = |state_machine: &mut StateMachine<_, _, _, _>| { state_machine .execute_using_consensus_failure_handler:: _, _, _>( - ExecutionManager::NativeWhenPossible, + ExecutionManager::new(ExecutionStrategyWithHandler::NativeWhenPossible), Some(|| { sp_externalities::with_externalities(|mut ext| { ext.register_extension(DummyExt(2)).unwrap(); diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index 45474a44693ab..51185480f1381 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -196,11 +196,21 @@ pub mod well_known_keys { /// Stored as a raw byte vector. Required by substrate. pub const CODE: &'static [u8] = b":code"; - /// Number of wasm linear memory pages required for execution of the runtime. + /// Number of wasm linear memory pages required for execution of the runtime in consensus + /// context. By consensus, we mean any code that contributes to the state transition and + /// therefore need to be part of consensus. /// /// The type of this value is encoded `u64`. + /// + /// Note that this value is used for consensus-related runtime operations, such as block import. pub const HEAP_PAGES: &'static [u8] = b":heappages"; + /// Number of wasm linear memory pages required for the execution of the runtime in + /// offchain context. By offchain we mean anything other than `consensus` (see [`HEAP_PAGES`]). + /// + /// The type of this value is encoded `u64`. + pub const OFFCHAIN_HEAP_PAGES: &'static [u8] = b":offchain_heappages"; + /// Current extrinsic index (u32) is stored under this key. pub const EXTRINSIC_INDEX: &'static [u8] = b":extrinsic_index"; diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index d08a01a4decbe..2cd8b7320a6e2 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -23,7 +23,7 @@ pub mod client_ext; pub use self::client_ext::{ClientBlockImportExt, ClientExt}; pub use sc_client_api::{ - execution_extensions::{ExecutionExtensions, ExecutionStrategies}, + execution_extensions::{ExecutionConfigs, ExecutionExtensions}, BadBlocks, ForkBlocks, }; pub use sc_client_db::{self, Backend}; @@ -35,7 +35,7 @@ pub use sp_keyring::{ }; pub use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; pub use sp_runtime::{Storage, StorageChild}; -pub use sp_state_machine::ExecutionStrategy; +pub use sp_state_machine::{ExecutionConfig, ExecutionStrategy}; use futures::{ future::{Future, FutureExt}, @@ -74,7 +74,7 @@ impl GenesisInit for () { /// A builder for creating a test client instance. pub struct TestClientBuilder { - execution_strategies: ExecutionStrategies, + execution_configs: ExecutionConfigs, genesis_init: G, /// The key is an unprefixed storage key, this only contains /// default child trie content. @@ -129,7 +129,7 @@ impl pub fn with_backend(backend: Arc) -> Self { TestClientBuilder { backend, - execution_strategies: ExecutionStrategies::default(), + execution_configs: ExecutionConfigs::default(), child_storage_extension: Default::default(), genesis_init: Default::default(), _executor: Default::default(), @@ -174,12 +174,13 @@ impl /// Set the execution strategy that should be used by all contexts. pub fn set_execution_strategy(mut self, execution_strategy: ExecutionStrategy) -> Self { - self.execution_strategies = ExecutionStrategies { - syncing: execution_strategy, - importing: execution_strategy, - block_construction: execution_strategy, - offchain_worker: execution_strategy, - other: execution_strategy, + self.execution_configs = ExecutionConfigs { + // NOTE: we set the context to consensus for all of them. + syncing: execution_strategy.in_consensus(), + importing: execution_strategy.in_consensus(), + block_construction: execution_strategy.in_consensus(), + offchain_worker: execution_strategy.in_offchain(), + other: execution_strategy.in_consensus(), }; self } @@ -244,7 +245,7 @@ impl self.fork_blocks, self.bad_blocks, ExecutionExtensions::new( - self.execution_strategies, + self.execution_configs, self.keystore, sc_offchain::OffchainDb::factory_from_backend(&*self.backend), ), @@ -287,7 +288,7 @@ impl { let executor = executor .into() - .unwrap_or_else(|| NativeExecutor::new(WasmExecutionMethod::Interpreted, None, 8)); + .unwrap_or_else(|| NativeExecutor::new(WasmExecutionMethod::Interpreted, 8)); let executor = LocalCallExecutor::new( self.backend.clone(), executor, diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index 3db433968c9f8..2ac42a60438e6 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -427,5 +427,5 @@ pub fn new_light_fetcher() -> LightFetcher { /// Create a new native executor. pub fn new_native_executor() -> sc_executor::NativeExecutor { - sc_executor::NativeExecutor::new(sc_executor::WasmExecutionMethod::Interpreted, None, 8) + sc_executor::NativeExecutor::new(sc_executor::WasmExecutionMethod::Interpreted, 8) } diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 316a553ed027d..ddb752b216ed4 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -362,7 +362,7 @@ mod tests { native_executor_instance!(NativeDispatch, crate::api::dispatch, crate::native_version); fn executor() -> NativeExecutor { - NativeExecutor::new(WasmExecutionMethod::Interpreted, None, 8) + NativeExecutor::new(WasmExecutionMethod::Interpreted, 8) } fn new_test_ext() -> TestExternalities { @@ -420,6 +420,7 @@ mod tests { let runtime_code = RuntimeCode { code_fetcher: &sp_core::traits::WrappedRuntimeCode(wasm_binary_unwrap().into()), hash: Vec::new(), + context: sp_core::traits::CodeContext::Consensus, heap_pages: None, }; @@ -529,6 +530,7 @@ mod tests { let runtime_code = RuntimeCode { code_fetcher: &sp_core::traits::WrappedRuntimeCode(wasm_binary_unwrap().into()), hash: Vec::new(), + context: sp_core::traits::CodeContext::Consensus, heap_pages: None, }; diff --git a/test-utils/test-runner/src/lib.rs b/test-utils/test-runner/src/lib.rs index 9f0a8d5d6cb6a..221948cbc2252 100644 --- a/test-utils/test-runner/src/lib.rs +++ b/test-utils/test-runner/src/lib.rs @@ -23,16 +23,17 @@ //! Allows you to test //!
//! -//! - Migrations -//! - Runtime Upgrades -//! - Pallets and general runtime functionality. +//! - Migrations +//! - Runtime Upgrades +//! - Pallets and general runtime functionality. //! //! This works by running a full node with a Manual Seal-BABE™ hybrid consensus for block authoring. //! //!

Note

-//! The running node has no signature verification, which allows us author extrinsics for any account on chain. -//!
-//!
+//! The running node has no signature verification, which allows us author extrinsics for any +//! account on chain. +//! +//! //! //!

How do I Use this?

//! @@ -56,7 +57,7 @@ //! use sp_runtime::{traits::IdentifyAccount, MultiSigner, generic::Era}; //! use sc_executor::WasmExecutionMethod; //! use sc_network::{multiaddr, config::TransportConfig}; -//! use sc_client_api::execution_extensions::ExecutionStrategies; +//! use sc_client_api::execution_extensions::ExecutionConfigs; //! use sc_informant::OutputFormat; //! use sp_api::TransactionFor; //! @@ -187,7 +188,7 @@ //! fn simple_balances_test() { //! // given //! let config = NodeConfig { -//! execution_strategies: ExecutionStrategies { +//! execution_configs: ExecutionConfigs { //! syncing: sc_client_api::ExecutionStrategy::NativeWhenPossible, //! importing: sc_client_api::ExecutionStrategy::NativeWhenPossible, //! block_construction: sc_client_api::ExecutionStrategy::NativeWhenPossible, diff --git a/test-utils/test-runner/src/utils.rs b/test-utils/test-runner/src/utils.rs index e0176fcb6cc29..eaacaadca73f6 100644 --- a/test-utils/test-runner/src/utils.rs +++ b/test-utils/test-runner/src/utils.rs @@ -17,7 +17,7 @@ // along with this program. If not, see . use futures::FutureExt; -use sc_client_api::execution_extensions::ExecutionStrategies; +use sc_client_api::execution_extensions::ExecutionConfigs; use sc_executor::WasmExecutionMethod; use sc_informant::OutputFormat; use sc_network::{ @@ -84,12 +84,22 @@ pub fn default_config( state_cache_child_ratio: None, chain_spec, wasm_method: WasmExecutionMethod::Interpreted, - execution_strategies: ExecutionStrategies { - syncing: sc_client_api::ExecutionStrategy::AlwaysWasm, - importing: sc_client_api::ExecutionStrategy::AlwaysWasm, - block_construction: sc_client_api::ExecutionStrategy::AlwaysWasm, - offchain_worker: sc_client_api::ExecutionStrategy::AlwaysWasm, - other: sc_client_api::ExecutionStrategy::AlwaysWasm, + execution_configs: ExecutionConfigs { + syncing: sc_client_api::ExecutionConfig::new_offchain( + sc_client_api::ExecutionStrategy::AlwaysWasm, + ), + importing: sc_client_api::ExecutionConfig::new_offchain( + sc_client_api::ExecutionStrategy::AlwaysWasm, + ), + block_construction: sc_client_api::ExecutionConfig::new_offchain( + sc_client_api::ExecutionStrategy::AlwaysWasm, + ), + offchain_worker: sc_client_api::ExecutionConfig::new_offchain( + sc_client_api::ExecutionStrategy::AlwaysWasm, + ), + other: sc_client_api::ExecutionConfig::new_offchain( + sc_client_api::ExecutionStrategy::AlwaysWasm, + ), }, rpc_http: None, rpc_ws: None, @@ -102,7 +112,6 @@ pub fn default_config( prometheus_config: None, telemetry_endpoints: None, telemetry_external_transport: None, - default_heap_pages: None, offchain_worker: Default::default(), force_authoring: false, disable_grandpa: false, diff --git a/utils/browser/src/lib.rs b/utils/browser/src/lib.rs index 0870ea84296c0..c8229881288e5 100644 --- a/utils/browser/src/lib.rs +++ b/utils/browser/src/lib.rs @@ -87,10 +87,9 @@ where }, keystore_remote: Default::default(), keystore: KeystoreConfig::InMemory, - default_heap_pages: Default::default(), dev_key_seed: Default::default(), disable_grandpa: Default::default(), - execution_strategies: Default::default(), + execution_configs: Default::default(), force_authoring: Default::default(), impl_name: String::from("parity-substrate"), impl_version: String::from("0.0.0"), diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 925cfd07d03e2..03ed7cc6ba713 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -113,7 +113,7 @@ impl BenchmarkCmd { let state = BenchmarkingState::::new(genesis_storage, cache_size, self.record_proof)?; let executor = NativeExecutor::::new( wasm_method, - self.heap_pages, + // TODO: set the heap pages via state alteration 2, // The runtime instances cache size. ); @@ -137,10 +137,11 @@ impl BenchmarkCmd { "Benchmark_benchmark_metadata", &(self.extra).encode(), extensions(), - &sp_state_machine::backend::BackendRuntimeCode::new(&state).runtime_code()?, + &sp_state_machine::backend::BackendRuntimeCode::new(&state) + .runtime_code(sp_core::traits::CodeContext::Consensus)?, sp_core::testing::TaskExecutor::new(), ) - .execute(strategy.into()) + .execute(sp_state_machine::ExecutionConfig::new_offchain(strategy.into())) .map_err(|e| format!("Error getting benchmark list: {:?}", e))?; let (list, storage_info) = @@ -193,10 +194,10 @@ impl BenchmarkCmd { .encode(), extensions(), &sp_state_machine::backend::BackendRuntimeCode::new(&state) - .runtime_code()?, + .runtime_code(sp_core::traits::CodeContext::Consensus)?, sp_core::testing::TaskExecutor::new(), ) - .execute(strategy.into()) + .execute(sp_state_machine::ExecutionConfig::new_offchain(strategy.into())) .map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?; let batch = diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 4f31bd741b3a0..320cecb60bd22 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -186,13 +186,12 @@ where ExecDispatch: NativeExecutionDispatch + 'static, { let wasm_method = shared.wasm_method; - let execution = shared.execution; - let heap_pages = shared.heap_pages.or(config.default_heap_pages); + let execution: sp_state_machine::ExecutionStrategy = shared.execution.into(); + // TODO: set the heap pages. let mut changes = Default::default(); let max_runtime_instances = config.max_runtime_instances; - let executor = - NativeExecutor::::new(wasm_method.into(), heap_pages, max_runtime_instances); + let executor = NativeExecutor::::new(wasm_method.into(), max_runtime_instances); let ext = { let builder = match command.state { @@ -226,10 +225,11 @@ where "TryRuntime_on_runtime_upgrade", &[], ext.extensions, - &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend).runtime_code()?, + &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) + .runtime_code(sp_core::traits::CodeContext::Consensus)?, sp_core::testing::TaskExecutor::new(), ) - .execute(execution.into()) + .execute(execution.in_consensus()) .map_err(|e| format!("failed to execute 'TryRuntime_on_runtime_upgrade': {:?}", e))?; let (weight, total_weight) = <(u64, u64) as Decode>::decode(&mut &*encoded_result) @@ -259,13 +259,12 @@ where ExecDispatch: NativeExecutionDispatch + 'static, { let wasm_method = shared.wasm_method; - let execution = shared.execution; - let heap_pages = shared.heap_pages.or(config.default_heap_pages); + let execution: sp_state_machine::ExecutionStrategy = shared.execution.into(); + // TODO: set heap pages. let mut changes = Default::default(); let max_runtime_instances = config.max_runtime_instances; - let executor = - NativeExecutor::::new(wasm_method.into(), heap_pages, max_runtime_instances); + let executor = NativeExecutor::::new(wasm_method.into(), max_runtime_instances); let mode = match command.state { State::Live { snapshot_path, modules } => { @@ -315,10 +314,11 @@ where "OffchainWorkerApi_offchain_worker", header.encode().as_ref(), ext.extensions, - &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend).runtime_code()?, + &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) + .runtime_code(sp_core::traits::CodeContext::Offchain)?, sp_core::testing::TaskExecutor::new(), ) - .execute(execution.into()) + .execute(execution.in_offchain()) .map_err(|e| format!("failed to execute 'OffchainWorkerApi_offchain_worker': {:?}", e))?; log::info!("OffchainWorkerApi_offchain_worker executed without errors."); @@ -340,13 +340,12 @@ where ExecDispatch: NativeExecutionDispatch + 'static, { let wasm_method = shared.wasm_method; - let execution = shared.execution; - let heap_pages = shared.heap_pages.or(config.default_heap_pages); + let execution: sp_state_machine::ExecutionStrategy = shared.execution.into(); + // TODO: set heap pages. let mut changes = Default::default(); let max_runtime_instances = config.max_runtime_instances; - let executor = - NativeExecutor::::new(wasm_method.into(), heap_pages, max_runtime_instances); + let executor = NativeExecutor::::new(wasm_method.into(), max_runtime_instances); let block_hash = shared.block_at::()?; let block: Block = rpc_api::get_block::(shared.url.clone(), block_hash).await?; @@ -410,10 +409,11 @@ where "Core_execute_block", block.encode().as_ref(), ext.extensions, - &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend).runtime_code()?, + &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) + .runtime_code(sp_core::traits::CodeContext::Consensus)?, sp_core::testing::TaskExecutor::new(), ) - .execute(execution.into()) + .execute(execution.in_consensus()) .map_err(|e| format!("failed to execute 'Core_execute_block': {:?}", e))?; debug_assert!(_encoded_result == vec![1]);