diff --git a/.gitignore b/.gitignore index c8f1ea9567bc2..ce302c74e10a0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ rls*.log **/hfuzz_workspace/ .cargo/ .cargo-remote.toml +*.bin diff --git a/Cargo.lock b/Cargo.lock index 58c6baeb23712..dea99ed63bdc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1861,6 +1861,17 @@ dependencies = [ "sp-api", ] +[[package]] +name = "frame-try-runtime" +version = "0.9.0" +dependencies = [ + "frame-support", + "parity-scale-codec", + "sp-api", + "sp-runtime", + "sp-std", +] + [[package]] name = "fs-swap" version = "0.2.5" @@ -3949,6 +3960,7 @@ dependencies = [ "substrate-build-script-utils", "substrate-frame-cli", "tempfile", + "try-runtime-cli", "wasm-bindgen", "wasm-bindgen-futures", ] @@ -4072,6 +4084,7 @@ dependencies = [ "frame-system", "frame-system-benchmarking", "frame-system-rpc-runtime-api", + "frame-try-runtime", "hex-literal", "node-primitives", "pallet-assets", @@ -6372,6 +6385,24 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "remote-externalities" +version = "0.9.0" +dependencies = [ + "async-std", + "bincode", + "env_logger 0.8.2", + "futures 0.1.30", + "hex-literal", + "jsonrpc-core-client", + "log", + "sc-rpc", + "sc-rpc-api", + "sp-core", + "sp-io", + "tokio 0.1.22", +] + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -9898,6 +9929,27 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "try-runtime-cli" +version = "0.9.0" +dependencies = [ + "frame-try-runtime", + "log", + "parity-scale-codec", + "remote-externalities", + "sc-cli", + "sc-client-api", + "sc-executor", + "sc-service", + "sp-api", + "sp-blockchain", + "sp-core", + "sp-externalities", + "sp-runtime", + "sp-state-machine", + "structopt", +] + [[package]] name = "trybuild" version = "1.0.39" diff --git a/Cargo.toml b/Cargo.toml index 38b3a2bdcf296..adc8960ffd765 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ members = [ "frame/contracts/rpc", "frame/contracts/rpc/runtime-api", "frame/democracy", + "frame/try-runtime", "frame/elections", "frame/example", "frame/example-offchain-worker", @@ -185,7 +186,9 @@ members = [ "utils/build-script-utils", "utils/fork-tree", "utils/frame/benchmarking-cli", + "utils/frame/remote-externalities", "utils/frame/frame-utilities-cli", + "utils/frame/try-runtime/cli", "utils/frame/rpc/support", "utils/frame/rpc/system", "utils/prometheus", diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 6162726c8947d..4aa73e2f70607 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -99,6 +99,7 @@ node-executor = { version = "2.0.0", path = "../executor" } sc-cli = { version = "0.9.0", optional = true, path = "../../../client/cli" } frame-benchmarking-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/benchmarking-cli" } node-inspect = { version = "0.8.0", optional = true, path = "../inspect" } +try-runtime-cli = { version = "0.9.0", optional = true, path = "../../../utils/frame/try-runtime/cli" } # WASM-specific dependencies wasm-bindgen = { version = "0.2.57", optional = true } @@ -133,6 +134,7 @@ node-inspect = { version = "0.8.0", optional = true, path = "../inspect" } frame-benchmarking-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/benchmarking-cli" } substrate-build-script-utils = { version = "3.0.0", optional = true, path = "../../../utils/build-script-utils" } substrate-frame-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/frame-utilities-cli" } +try-runtime-cli = { version = "0.9.0", optional = true, path = "../../../utils/frame/try-runtime/cli" } [build-dependencies.sc-cli] version = "0.9.0" @@ -157,8 +159,15 @@ cli = [ "sc-finality-grandpa-warp-sync", "structopt", "substrate-build-script-utils", + "try-runtime-cli", ] runtime-benchmarks = [ "node-runtime/runtime-benchmarks", "frame-benchmarking-cli", ] +# Enable features that allow the runtime to be tried and debugged. Name might be subject to change +# in the near future. +try-runtime = [ + "node-runtime/try-runtime", + "try-runtime-cli", +] diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 63a07e00e2197..9b80a3e345290 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -47,6 +47,11 @@ pub enum Subcommand { #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), + /// Try some experimental command on the runtime. This includes migration and runtime-upgrade + /// testing. + #[cfg(feature = "try-runtime")] + TryRuntime(try_runtime_cli::TryRuntimeCmd), + /// Verify a signature for a message, provided on STDIN, with a given (public or secret) key. Verify(VerifyCmd), diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 461930a613d97..d3689bdcd6743 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -149,5 +149,20 @@ pub fn run() -> Result<()> { Ok((cmd.run(client, backend), task_manager)) }) }, + #[cfg(feature = "try-runtime")] + Some(Subcommand::TryRuntime(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + // we don't need any of the components of new_partial, just a runtime, or a task + // manager to do `async_run`. + let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry); + let task_manager = sc_service::TaskManager::new( + config.task_executor.clone(), + registry, + ).unwrap(); + + Ok((cmd.run::(config), task_manager)) + }) + } } } diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index f77a16a10f4c6..1a55efbf85157 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -43,6 +43,7 @@ frame-support = { version = "3.0.0", default-features = false, path = "../../../ frame-system = { version = "3.0.0", default-features = false, path = "../../../frame/system" } frame-system-benchmarking = { version = "3.0.0", default-features = false, path = "../../../frame/system/benchmarking", optional = true } frame-system-rpc-runtime-api = { version = "3.0.0", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } +frame-try-runtime = { version = "0.9.0", default-features = false, path = "../../../frame/try-runtime", optional = true } pallet-assets = { version = "3.0.0", default-features = false, path = "../../../frame/assets" } pallet-authority-discovery = { version = "3.0.0", default-features = false, path = "../../../frame/authority-discovery" } pallet-authorship = { version = "3.0.0", default-features = false, path = "../../../frame/authorship" } @@ -186,3 +187,7 @@ runtime-benchmarks = [ "frame-system-benchmarking", "hex-literal", ] +try-runtime = [ + "frame-executive/try-runtime", + "frame-try-runtime", +] diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 86e3075c3ae5d..fb2b189e2d379 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1080,7 +1080,14 @@ pub type SignedPayload = generic::SignedPayload; /// Extrinsic type that has already been checked. pub type CheckedExtrinsic = generic::CheckedExtrinsic; /// Executive: handles dispatch to the various modules. -pub type Executive = frame_executive::Executive, Runtime, AllModules>; +pub type Executive = frame_executive::Executive< + Runtime, + Block, + frame_system::ChainContext, + Runtime, + AllModules, + (), +>; /// MMR helper types. mod mmr { @@ -1325,15 +1332,24 @@ impl_runtime_apis! { } } + #[cfg(feature = "try-runtime")] + impl frame_try_runtime::TryRuntime for Runtime { + fn on_runtime_upgrade() -> Result<(Weight, Weight), sp_runtime::RuntimeString> { + frame_support::debug::RuntimeLogger::init(); + let weight = Executive::try_runtime_upgrade()?; + Ok((weight, RuntimeBlockWeights::get().max_block)) + } + } + #[cfg(feature = "runtime-benchmarks")] impl frame_benchmarking::Benchmark for Runtime { fn dispatch_benchmark( config: frame_benchmarking::BenchmarkConfig ) -> Result, sp_runtime::RuntimeString> { use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey}; - // Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues. - // To get around that, we separated the Session benchmarks into its own crate, which is why - // we need these two lines below. + // Trying to add benchmarks directly to the Session Pallet caused cyclic dependency + // issues. To get around that, we separated the Session benchmarks into its own crate, + // which is why we need these two lines below. use pallet_session_benchmarking::Module as SessionBench; use pallet_offences_benchmarking::Module as OffencesBench; use frame_system_benchmarking::Module as SystemBench; diff --git a/client/service/src/task_manager/mod.rs b/client/service/src/task_manager/mod.rs index 652e5d443977e..02d83e6dce7c1 100644 --- a/client/service/src/task_manager/mod.rs +++ b/client/service/src/task_manager/mod.rs @@ -234,7 +234,7 @@ pub struct TaskManager { impl TaskManager { /// If a Prometheus registry is passed, it will be used to report statistics about the /// service tasks. - pub(super) fn new( + pub fn new( executor: TaskExecutor, prometheus_registry: Option<&Registry>, ) -> Result { diff --git a/client/state-db/src/lib.rs b/client/state-db/src/lib.rs index dd2baf9d18ace..1f73f3cca35e9 100644 --- a/client/state-db/src/lib.rs +++ b/client/state-db/src/lib.rs @@ -24,12 +24,12 @@ //! Canonicalization window tracks a tree of blocks identified by header hash. The in-memory //! overlay allows to get any node that was inserted in any of the blocks within the window. //! The tree is journaled to the backing database and rebuilt on startup. -//! Canonicalization function selects one root from the top of the tree and discards all other roots and -//! their subtrees. +//! Canonicalization function selects one root from the top of the tree and discards all other roots +//! and their subtrees. //! //! # Pruning. -//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until pruning -//! constraints are satisfied. +//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until +//! pruning constraints are satisfied. mod noncanonical; mod pruning; diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml index 31f1f34174ed1..7ef00e7ff71c8 100644 --- a/frame/executive/Cargo.toml +++ b/frame/executive/Cargo.toml @@ -47,3 +47,6 @@ std = [ "sp-tracing/std", "sp-std/std", ] +try-runtime = [ + "frame-support/try-runtime" +] diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index b31f40dc28d6b..924adea95fd01 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -44,7 +44,8 @@ //! //! ## Usage //! -//! The default Substrate node template declares the [`Executive`](./struct.Executive.html) type in its library. +//! The default Substrate node template declares the [`Executive`](./struct.Executive.html) type in +//! its library. //! //! ### Example //! @@ -185,26 +186,58 @@ where } impl< - System: frame_system::Config, - Block: traits::Block, - Context: Default, - UnsignedValidator, - AllModules: - OnRuntimeUpgrade + - OnInitialize + - OnFinalize + - OffchainWorker, - COnRuntimeUpgrade: OnRuntimeUpgrade, -> Executive + System: frame_system::Config, + Block: traits::Block
, + Context: Default, + UnsignedValidator, + AllModules: OnRuntimeUpgrade + + OnInitialize + + OnFinalize + + OffchainWorker, + COnRuntimeUpgrade: OnRuntimeUpgrade, + > Executive where Block::Extrinsic: Checkable + Codec, - CheckedOf: - Applyable + - GetDispatchInfo, - CallOf: Dispatchable, + CheckedOf: Applyable + GetDispatchInfo, + CallOf: + Dispatchable, OriginOf: From>, - UnsignedValidator: ValidateUnsigned>, + UnsignedValidator: ValidateUnsigned>, { + /// Execute all `OnRuntimeUpgrade` of this runtime, and return the aggregate weight. + pub fn execute_on_runtime_upgrade() -> frame_support::weights::Weight { + let mut weight = 0; + weight = weight.saturating_add( + as OnRuntimeUpgrade>::on_runtime_upgrade(), + ); + weight = weight.saturating_add(COnRuntimeUpgrade::on_runtime_upgrade()); + weight = weight.saturating_add(::on_runtime_upgrade()); + + weight + } + + /// Execute all `OnRuntimeUpgrade` of this runtime, including the pre and post migration checks. + /// + /// This should only be used for testing. + #[cfg(feature = "try-runtime")] + pub fn try_runtime_upgrade() -> Result { + < + (frame_system::Module::, COnRuntimeUpgrade, AllModules) + as + OnRuntimeUpgrade + >::pre_upgrade()?; + + let weight = Self::execute_on_runtime_upgrade(); + + < + (frame_system::Module::, COnRuntimeUpgrade, AllModules) + as + OnRuntimeUpgrade + >::post_upgrade()?; + + Ok(weight) + } + /// Start the execution of a particular block. pub fn initialize_block(header: &System::Header) { sp_io::init_tracing(); @@ -234,10 +267,7 @@ where ) { let mut weight = 0; if Self::runtime_upgraded() { - // System is not part of `AllModules`, so we need to call this manually. - weight = weight.saturating_add( as OnRuntimeUpgrade>::on_runtime_upgrade()); - weight = weight.saturating_add(COnRuntimeUpgrade::on_runtime_upgrade()); - weight = weight.saturating_add(::on_runtime_upgrade()); + weight = weight.saturating_add(Self::execute_on_runtime_upgrade()); } >::initialize( block_number, diff --git a/frame/support/Cargo.toml b/frame/support/Cargo.toml index 8edf1ff6ddadc..b77907721be22 100644 --- a/frame/support/Cargo.toml +++ b/frame/support/Cargo.toml @@ -60,3 +60,4 @@ std = [ nightly = [] strict = [] runtime-benchmarks = [] +try-runtime = [] diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 106ec10c6c4ee..602bc1aa1a690 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -1547,7 +1547,25 @@ pub trait OnRuntimeUpgrade { /// block local data are not accessible. /// /// Return the non-negotiable weight consumed for runtime upgrade. - fn on_runtime_upgrade() -> crate::weights::Weight { 0 } + fn on_runtime_upgrade() -> crate::weights::Weight { + 0 + } + + /// Execute some pre-checks prior to a runtime upgrade. + /// + /// This hook is never meant to be executed on-chain but is meant to be used by testing tools. + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + Ok(()) + } + + /// Execute some post-checks after a runtime upgrade. + /// + /// This hook is never meant to be executed on-chain but is meant to be used by testing tools. + #[cfg(feature = "try-runtime")] + fn post_upgrade() -> Result<(), &'static str> { + Ok(()) + } } #[impl_for_tuples(30)] @@ -1557,6 +1575,20 @@ impl OnRuntimeUpgrade for Tuple { for_tuples!( #( weight = weight.saturating_add(Tuple::on_runtime_upgrade()); )* ); weight } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + let mut result = Ok(()); + for_tuples!( #( result = result.and(Tuple::pre_upgrade()); )* ); + result + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade() -> Result<(), &'static str> { + let mut result = Ok(()); + for_tuples!( #( result = result.and(Tuple::post_upgrade()); )* ); + result + } } /// Off-chain computation trait. diff --git a/frame/try-runtime/Cargo.toml b/frame/try-runtime/Cargo.toml new file mode 100644 index 0000000000000..9c1919d380b82 --- /dev/null +++ b/frame/try-runtime/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "frame-try-runtime" +version = "0.9.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for democracy" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } + +sp-api = { version = "3.0.0", path = "../../primitives/api", default-features = false } +sp-std = { version = "3.0.0", path = "../../primitives/std" , default-features = false } +sp-runtime = { version = "3.0.0", path = "../../primitives/runtime" , default-features = false } + +frame-support = { version = "3.0.0", path = "../support", default-features = false } + +[features] +default = [ "std" ] +std = [ + "sp-api/std", + "sp-std/std", + "sp-runtime/std", + "frame-support/std", +] diff --git a/frame/try-runtime/src/lib.rs b/frame/try-runtime/src/lib.rs new file mode 100644 index 0000000000000..dcd3a47878237 --- /dev/null +++ b/frame/try-runtime/src/lib.rs @@ -0,0 +1,37 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Supporting types for try-runtime, testing and dry-running commands. + +#![cfg_attr(not(feature = "std"), no_std)] + +use sp_std::prelude::*; +use frame_support::weights::Weight; + +sp_api::decl_runtime_apis! { + /// Runtime api for testing the execution of a runtime upgrade. + pub trait TryRuntime { + /// dry-run runtime upgrades, returning the total weight consumed. + /// + /// This should do EXACTLY the same operations as the runtime would have done in the case of + /// a runtime upgrade (e.g. pallet ordering must be the same) + /// + /// Returns the consumed weight of the migration in case of a successful one, combined with + /// the total allowed block weight of the runtime. + fn on_runtime_upgrade() -> Result<(Weight, Weight), sp_runtime::RuntimeString>; + } +} diff --git a/primitives/core/src/hexdisplay.rs b/primitives/core/src/hexdisplay.rs index 304b665a72c9b..e590eec0e5aec 100644 --- a/primitives/core/src/hexdisplay.rs +++ b/primitives/core/src/hexdisplay.rs @@ -71,6 +71,12 @@ impl AsBytesRef for sp_std::vec::Vec { fn as_bytes_ref(&self) -> &[u8] { &self } } +impl AsBytesRef for sp_storage::StorageKey { + fn as_bytes_ref(&self) -> &[u8] { + self.as_ref() + } +} + macro_rules! impl_non_endians { ( $( $t:ty ),* ) => { $( impl AsBytesRef for $t { diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index a6f9d06824642..5f10fc0a276c4 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -48,20 +48,22 @@ pub struct TestExternalities where H::Out: codec::Codec + Ord, { + /// The overlay changed storage. overlay: OverlayedChanges, offchain_db: TestPersistentOffchainDB, - storage_transaction_cache: StorageTransactionCache< - as Backend>::Transaction, H, N - >, - backend: InMemoryBackend, + storage_transaction_cache: + StorageTransactionCache< as Backend>::Transaction, H, N>, + /// Storage backend. + pub backend: InMemoryBackend, changes_trie_config: Option, changes_trie_storage: ChangesTrieInMemoryStorage, - extensions: Extensions, + /// Extensions. + pub extensions: Extensions, } impl TestExternalities - where - H::Out: Ord + 'static + codec::Codec +where + H::Out: Ord + 'static + codec::Codec, { /// Get externalities implementation. pub fn ext(&mut self) -> Ext> { diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index 1e9f9766072e4..1016b73eb1e3b 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -31,10 +31,15 @@ use codec::{Encode, Decode}; #[derive(PartialEq, Eq, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))] pub struct StorageKey( - #[cfg_attr(feature = "std", serde(with="impl_serde::serialize"))] - pub Vec, + #[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))] pub Vec, ); +impl AsRef<[u8]> for StorageKey { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + /// Storage key with read/write tracking information. #[derive(PartialEq, Eq, RuntimeDebug, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Hash, PartialOrd, Ord))] diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml new file mode 100644 index 0000000000000..41a3b26217860 --- /dev/null +++ b/utils/frame/remote-externalities/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "remote-externalities" +version = "0.9.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "An externalities provided environemnt that can load itself from remote nodes or cache files" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +jsonrpc-core-client = { version = "15.1.0", features = ["http"] } +sc-rpc-api = { version = "0.9.0", path = "../../../client/rpc-api" } +sc-rpc = { version = "3.0.0", path = "../../../client/rpc" } +futures = "0.1.29" + +hex-literal = "0.3.1" +env_logger = "0.8.2" +log = "0.4.11" +bincode = "1.3.1" +tokio = "0.1.22" + +sp-io = { version = "3.0.0", path = "../../../primitives/io" } +sp-core = { version = "3.0.0", path = "../../../primitives/core" } + +[dev-dependencies] +async-std = { version = "1.6.5", features = ["attributes"] } + +[features] +remote-test = [] diff --git a/utils/frame/remote-externalities/proxy_test b/utils/frame/remote-externalities/proxy_test new file mode 100644 index 0000000000000..adb93f5ba270c Binary files /dev/null and b/utils/frame/remote-externalities/proxy_test differ diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs new file mode 100644 index 0000000000000..6c8b49c7c85d5 --- /dev/null +++ b/utils/frame/remote-externalities/src/lib.rs @@ -0,0 +1,454 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Remote Externalities +//! +//! An equivalent of `sp_io::TestExternalities` that can load its state from a remote substrate +//! based chain, or a local cache file. +//! +//! #### Runtime to Test Against +//! +//! While not absolutely necessary, you most likely need a `Runtime` equivalent in your test setup +//! through which you can infer storage types. There are two options here: +//! +//! 1. Build a mock runtime, similar how to you would build one in a pallet test (see example +//! below). The very important point here is that this mock needs to hold real values for types +//! that matter for you, based on the chain of interest. Some typical ones are: +//! +//! - `sp_runtime::AccountId32` as `AccountId`. +//! - `u32` as `BlockNumber`. +//! - `u128` as Balance. +//! +//! Once you have your `Runtime`, you can use it for storage type resolution and do things like +//! `>::storage_getter()` or `>::get()`. +//! +//! 2. Or, you can use a real runtime. +//! +//! ### Example +//! +//! With a test runtime +//! +//! ```ignore +//! use remote_externalities::Builder; +//! +//! #[derive(Clone, Eq, PartialEq, Debug, Default)] +//! pub struct TestRuntime; +//! +//! use frame_system as system; +//! impl_outer_origin! { +//! pub enum Origin for TestRuntime {} +//! } +//! +//! impl frame_system::Config for TestRuntime { +//! .. +//! // we only care about these two for now. The rest can be mock. The block number type of +//! // kusama is u32. +//! type BlockNumber = u32; +//! type Header = Header; +//! .. +//! } +//! +//! #[test] +//! fn test_runtime_works() { +//! let hash: Hash = +//! hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into(); +//! let parent: Hash = +//! hex!["540922e96a8fcaf945ed23c6f09c3e189bd88504ec945cc2171deaebeaf2f37e"].into(); +//! Builder::new() +//! .at(hash) +//! .module("System") +//! .build() +//! .execute_with(|| { +//! assert_eq!( +//! // note: the hash corresponds to 3098546. We can check only the parent. +//! // https://polkascan.io/kusama/block/3098546 +//! >::block_hash(3098545u32), +//! parent, +//! ) +//! }); +//! } +//! ``` +//! +//! Or with the real kusama runtime. +//! +//! ```ignore +//! use remote_externalities::Builder; +//! use kusama_runtime::Runtime; +//! +//! #[test] +//! fn test_runtime_works() { +//! let hash: Hash = +//! hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into(); +//! Builder::new() +//! .at(hash) +//! .module("Staking") +//! .build() +//! .execute_with(|| assert_eq!(>::validator_count(), 400)); +//! } +//! ``` + +use std::{ + fs, + path::{Path, PathBuf}, +}; +use log::*; +use sp_core::{hashing::twox_128}; +pub use sp_io::TestExternalities; +use sp_core::{ + hexdisplay::HexDisplay, + storage::{StorageKey, StorageData}, +}; +use futures::future::Future; + +type KeyPair = (StorageKey, StorageData); +type Number = u32; +type Hash = sp_core::H256; +// TODO: make these two generic. + +const LOG_TARGET: &'static str = "remote-ext"; + +/// The execution mode. +#[derive(Clone)] +pub enum Mode { + /// Online. + Online(OnlineConfig), + /// Offline. Uses a cached file and needs not any client config. + Offline(OfflineConfig), +} + +/// configuration of the online execution. +/// +/// A cache config must be present. +#[derive(Clone)] +pub struct OfflineConfig { + /// The configuration of the cache file to use. It must be present. + pub cache: CacheConfig, +} + +/// Configuration of the online execution. +/// +/// A cache config may be present and will be written to in that case. +#[derive(Clone)] +pub struct OnlineConfig { + /// The HTTP uri to use. + pub uri: String, + /// The block number at which to connect. Will be latest finalized head if not provided. + pub at: Option, + /// An optional cache file to WRITE to, not for reading. Not cached if set to `None`. + pub cache: Option, + /// The modules to scrape. If empty, entire chain state will be scraped. + pub modules: Vec, +} + +impl Default for OnlineConfig { + fn default() -> Self { + Self { + uri: "http://localhost:9933".into(), + at: None, + cache: None, + modules: Default::default(), + } + } +} + +/// Configuration of the cache. +#[derive(Clone)] +pub struct CacheConfig { + // TODO: I could mix these two into one filed, but I think separate is better bc one can be + // configurable while one not. + /// File name. + pub name: String, + /// Base directory. + pub directory: String, +} + +impl Default for CacheConfig { + fn default() -> Self { + Self { name: "CACHE".into(), directory: ".".into() } + } +} + +impl CacheConfig { + fn path(&self) -> PathBuf { + Path::new(&self.directory).join(self.name.clone()) + } +} + +/// Builder for remote-externalities. +pub struct Builder { + inject: Vec, + mode: Mode, + chain: String, +} + +impl Default for Builder { + fn default() -> Self { + Self { + inject: Default::default(), + mode: Mode::Online(OnlineConfig { + at: None, + uri: "http://localhost:9933".into(), + cache: None, + modules: Default::default(), + }), + chain: "UNSET".into(), + } + } +} + +// Mode methods +impl Builder { + fn as_online(&self) -> &OnlineConfig { + match &self.mode { + Mode::Online(config) => &config, + _ => panic!("Unexpected mode: Online"), + } + } + + fn as_online_mut(&mut self) -> &mut OnlineConfig { + match &mut self.mode { + Mode::Online(config) => config, + _ => panic!("Unexpected mode: Online"), + } + } +} + +// RPC methods +impl Builder { + async fn rpc_get_head(&self) -> Hash { + let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + let uri = self.as_online().uri.clone(); + rt.block_on::<_, _, ()>(futures::lazy(move || { + trace!(target: LOG_TARGET, "rpc: finalized_head"); + let client: sc_rpc_api::chain::ChainClient = + jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); + Ok(client.finalized_head().wait().unwrap()) + })) + .unwrap() + } + + /// Relay the request to `state_getPairs` rpc endpoint. + /// + /// Note that this is an unsafe RPC. + async fn rpc_get_pairs(&self, prefix: StorageKey, at: Hash) -> Vec { + let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + let uri = self.as_online().uri.clone(); + rt.block_on::<_, _, ()>(futures::lazy(move || { + trace!(target: LOG_TARGET, "rpc: storage_pairs: {:?} / {:?}", prefix, at); + let client: sc_rpc_api::state::StateClient = + jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); + Ok(client.storage_pairs(prefix, Some(at)).wait().unwrap()) + })) + .unwrap() + } + + /// Get the chain name. + async fn chain_name(&self) -> String { + let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + let uri = self.as_online().uri.clone(); + rt.block_on::<_, _, ()>(futures::lazy(move || { + trace!(target: LOG_TARGET, "rpc: system_chain"); + let client: sc_rpc_api::system::SystemClient<(), ()> = + jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); + Ok(client.system_chain().wait().unwrap()) + })) + .unwrap() + } +} + +// Internal methods +impl Builder { + /// Save the given data as cache. + fn save_cache(&self, data: &[KeyPair], path: &Path) { + let bdata = bincode::serialize(data).unwrap(); + info!(target: LOG_TARGET, "writing to cache file {:?}", path); + fs::write(path, bdata).unwrap(); + } + + /// initialize `Self` from cache. Panics if the file does not exist. + fn load_cache(&self, path: &Path) -> Vec { + info!(target: LOG_TARGET, "scraping keypairs from cache {:?}", path,); + let bytes = fs::read(path).unwrap(); + bincode::deserialize(&bytes[..]).unwrap() + } + + /// Build `Self` from a network node denoted by `uri`. + async fn load_remote(&self) -> Vec { + let config = self.as_online(); + let at = self.as_online().at.unwrap().clone(); + info!(target: LOG_TARGET, "scraping keypairs from remote node {} @ {:?}", config.uri, at); + + let keys_and_values = if config.modules.len() > 0 { + let mut filtered_kv = vec![]; + for f in config.modules.iter() { + let hashed_prefix = StorageKey(twox_128(f.as_bytes()).to_vec()); + let module_kv = self.rpc_get_pairs(hashed_prefix.clone(), at).await; + info!( + target: LOG_TARGET, + "downloaded data for module {} (count: {} / prefix: {:?}).", + f, + module_kv.len(), + HexDisplay::from(&hashed_prefix), + ); + filtered_kv.extend(module_kv); + } + filtered_kv + } else { + info!(target: LOG_TARGET, "downloading data for all modules."); + self.rpc_get_pairs(StorageKey(vec![]), at).await.into_iter().collect::>() + }; + + keys_and_values + } + + async fn init_remote_client(&mut self) { + self.as_online_mut().at = Some(self.rpc_get_head().await); + self.chain = self.chain_name().await; + } + + async fn pre_build(mut self) -> Vec { + let mut base_kv = match self.mode.clone() { + Mode::Offline(config) => self.load_cache(&config.cache.path()), + Mode::Online(config) => { + self.init_remote_client().await; + let kp = self.load_remote().await; + if let Some(c) = config.cache { + self.save_cache(&kp, &c.path()); + } + kp + } + }; + + info!( + target: LOG_TARGET, + "extending externalities with {} manually injected keys", + self.inject.len() + ); + base_kv.extend(self.inject.clone()); + base_kv + } +} + +// Public methods +impl Builder { + /// Create a new builder. + pub fn new() -> Self { + Default::default() + } + + /// Inject a manual list of key and values to the storage. + pub fn inject(mut self, injections: &[KeyPair]) -> Self { + for i in injections { + self.inject.push(i.clone()); + } + self + } + + /// Configure a cache to be used. + pub fn mode(mut self, mode: Mode) -> Self { + self.mode = mode; + self + } + + /// Build the test externalities. + pub async fn build(self) -> TestExternalities { + let kv = self.pre_build().await; + let mut ext = TestExternalities::new_empty(); + + info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len()); + for (k, v) in kv { + let (k, v) = (k.0, v.0); + ext.insert(k, v); + } + ext + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn init_logger() { + let _ = env_logger::Builder::from_default_env() + .format_module_path(false) + .format_level(true) + .try_init(); + } + + #[async_std::test] + #[cfg(feature = "remote-test")] + async fn can_build_one_pallet() { + init_logger(); + Builder::new() + .mode(Mode::Online(OnlineConfig { + modules: vec!["Proxy".into()], + ..Default::default() + })) + .build() + .await + .execute_with(|| {}); + } + + #[async_std::test] + async fn can_load_cache() { + init_logger(); + Builder::new() + .mode(Mode::Offline(OfflineConfig { + cache: CacheConfig { name: "proxy_test".into(), ..Default::default() }, + })) + .build() + .await + .execute_with(|| {}); + } + + #[async_std::test] + #[cfg(feature = "remote-test")] + async fn can_create_cache() { + init_logger(); + Builder::new() + .mode(Mode::Online(OnlineConfig { + cache: Some(CacheConfig { + name: "test_cache_to_remove.bin".into(), + ..Default::default() + }), + ..Default::default() + })) + .build() + .await + .execute_with(|| {}); + + let to_delete = std::fs::read_dir(CacheConfig::default().directory) + .unwrap() + .into_iter() + .map(|d| d.unwrap()) + .filter(|p| p.path().extension().unwrap_or_default() == "bin") + .collect::>(); + + assert!(to_delete.len() > 0); + + for d in to_delete { + std::fs::remove_file(d.path()).unwrap(); + } + } + + #[async_std::test] + #[cfg(feature = "remote-test")] + async fn can_build_all() { + init_logger(); + Builder::new().build().await.execute_with(|| {}); + } +} diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml new file mode 100644 index 0000000000000..592d0a5b99d27 --- /dev/null +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "try-runtime-cli" +version = "0.9.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Cli command runtime testing and dry-running" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +log = "0.4.8" +parity-scale-codec = { version = "2.0.0" } + +sc-service = { version = "0.9.0", default-features = false, path = "../../../../client/service" } +sc-cli = { version = "0.9.0", path = "../../../../client/cli" } +sc-executor = { path = "../../../../client/executor" } +sc-client-api = { version = "3.0.0", path = "../../../../client/api" } +structopt = "0.3.8" +sp-state-machine = { version = "0.9.0", path = "../../../../primitives/state-machine" } +sp-api = { version = "3.0.0", path = "../../../../primitives/api" } +sp-blockchain = { version = "3.0.0", path = "../../../../primitives/blockchain" } +sp-runtime = { version = "3.0.0", path = "../../../../primitives/runtime" } +sp-externalities = { version = "0.9.0", path = "../../../../primitives/externalities" } +sp-core = { version = "3.0.0", path = "../../../../primitives/core" } +frame-try-runtime = { version = "0.9.0", path = "../../../../frame/try-runtime" } + +remote-externalities = { path = "../../remote-externalities" } diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs new file mode 100644 index 0000000000000..92526379f471e --- /dev/null +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -0,0 +1,178 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! `Structopt`-ready struct for `try-runtime`. + +use parity_scale_codec::Decode; +use std::{fmt::Debug, str::FromStr}; +use sc_service::Configuration; +use sc_cli::{CliConfiguration, ExecutionStrategy, WasmExecutionMethod}; +use sc_executor::NativeExecutor; +use sc_service::NativeExecutionDispatch; +use sp_state_machine::StateMachine; +use sp_runtime::traits::{Block as BlockT, NumberFor}; +use sp_core::storage::{StorageData, StorageKey, well_known_keys}; + +/// Various commands to try out the new runtime, over configurable states. +/// +/// For now this only assumes running the `on_runtime_upgrade` hooks. +#[derive(Debug, structopt::StructOpt)] +pub struct TryRuntimeCmd { + /// The shared parameters + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: sc_cli::SharedParams, + + /// The state to use to run the migration. Should be a valid FILE or HTTP URI. + #[structopt(short, long, default_value = "http://localhost:9933")] + pub state: State, + + /// The execution strategy that should be used for benchmarks + #[structopt( + long = "execution", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = "Native", + )] + pub execution: ExecutionStrategy, + + /// Method for executing Wasm runtime code. + #[structopt( + long = "wasm-execution", + value_name = "METHOD", + possible_values = &WasmExecutionMethod::enabled_variants(), + case_insensitive = true, + default_value = "Interpreted" + )] + pub wasm_method: WasmExecutionMethod, +} + +/// The state to use for a migration dry-run. +#[derive(Debug)] +pub enum State { + /// A snapshot. Inner value is a file path. + Snap(String), + + /// A live chain. Inner value is the HTTP uri. + Live(String), +} + +impl FromStr for State { + type Err = &'static str; + fn from_str(s: &str) -> Result { + match s.get(..7) { + // could use Url crate as well, but lets keep it simple for now. + Some("http://") => Ok(State::Live(s.to_string())), + Some("file://") => s + .split("//") + .collect::>() + .get(1) + .map(|s| State::Snap(s.to_string())) + .ok_or("invalid file URI"), + _ => Err("invalid format. Must be a valid HTTP or File URI"), + } + } +} + +impl TryRuntimeCmd { + pub async fn run(&self, config: Configuration) -> sc_cli::Result<()> + where + B: BlockT, + ExecDispatch: NativeExecutionDispatch + 'static, + { + let spec = config.chain_spec; + let genesis_storage = spec.build_storage()?; + + let code = StorageData( + genesis_storage + .top + .get(well_known_keys::CODE) + .expect("code key must exist in genesis storage; qed") + .to_vec(), + ); + let code_key = StorageKey(well_known_keys::CODE.to_vec()); + + let wasm_method = self.wasm_method; + let execution = self.execution; + + let mut changes = Default::default(); + // don't really care about these -- use the default values. + let max_runtime_instances = config.max_runtime_instances; + let heap_pages = config.default_heap_pages; + let executor = NativeExecutor::::new( + wasm_method.into(), + heap_pages, + max_runtime_instances, + ); + + let ext = { + use remote_externalities::{Builder, Mode, CacheConfig, OfflineConfig, OnlineConfig}; + let builder = match &self.state { + State::Snap(file_path) => Builder::new().mode(Mode::Offline(OfflineConfig { + cache: CacheConfig { name: file_path.into(), ..Default::default() }, + })), + State::Live(http_uri) => Builder::new().mode(Mode::Online(OnlineConfig { + uri: http_uri.into(), + ..Default::default() + })), + }; + + // inject the code into this ext. + builder.inject(&[(code_key, code)]).build().await + }; + + let encoded_result = StateMachine::<_, _, NumberFor, _>::new( + &ext.backend, + None, + &mut changes, + &executor, + "TryRuntime_on_runtime_upgrade", + &[], + ext.extensions, + &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) + .runtime_code()?, + sp_core::testing::TaskExecutor::new(), + ) + .execute(execution.into()) + .map_err(|e| format!("failed to execute 'TryRuntime_on_runtime_upgrade' due to {:?}", e))?; + + let (weight, total_weight) = <(u64, u64) as Decode>::decode(&mut &*encoded_result) + .map_err(|e| format!("failed to decode output due to {:?}", e))?; + log::info!( + "try-runtime executed without errors. Consumed weight = {}, total weight = {} ({})", + weight, + total_weight, + weight as f64 / total_weight as f64 + ); + + Ok(()) + } +} + +impl CliConfiguration for TryRuntimeCmd { + fn shared_params(&self) -> &sc_cli::SharedParams { + &self.shared_params + } + + fn chain_id(&self, _is_dev: bool) -> sc_cli::Result { + Ok(match self.shared_params.chain { + Some(ref chain) => chain.clone(), + None => "dev".into(), + }) + } +}